Instruction:
Make a summary of the following Python 3 code
def get_data_file_names_from_scan_base(scan_base, filter_str=['_analyzed.h5', '_interpreted.h5', '_cut.h5', '_result.h5', '_hists.h5'], sort_by_time=True, meta_data_v2=True):
    data_files = []
    if scan_base is None:
        return data_files
    if isinstance(scan_base, basestring):
        scan_base = [scan_base]
    for scan_base_str in scan_base:
        if '.h5' == os.path.splitext(scan_base_str)[1]:
            data_files.append(scan_base_str)
        else:
            data_files.extend(glob.glob(scan_base_str + '*.h5'))

    if filter_str:
        if isinstance(filter_str, basestring):
            filter_str = [filter_str]
        data_files = filter(lambda data_file: not any([(True if x in data_file else False) for x in filter_str]), data_files)
    if sort_by_time and len(data_files) > 1:
        f_list = {}
        for data_file in data_files:
            with tb.open_file(data_file, mode="r") as h5_file:
                try:
                    meta_data = h5_file.root.meta_data
                except tb.NoSuchNodeError:
                    logging.warning("File %s is missing meta_data" % h5_file.filename)
                else:
                    try:
                        if meta_data_v2:
                            timestamp = meta_data[0]["timestamp_start"]
                        else:
                            timestamp = meta_data[0]["timestamp"]
                    except IndexError:
                        logging.info("File %s has empty meta_data" % h5_file.filename)
                    else:
                        f_list[data_file] = timestamp

        data_files = list(sorted(f_list, key=f_list.__getitem__, reverse=False))
    return data_files

Response: Generate a list of. h5 files which have a similar file name.


Instruction:
Here you have a function in Python 3, explain what it does
def get_parameter_from_files(files, parameters=None, unique=False, sort=True):
    ''' Takes a list of files, searches for the parameter name in the file name and in the file.
    Returns a ordered dict with the file name in the first dimension and the corresponding parameter values in the second.
    If a scan parameter appears in the file name and in the file the first parameter setting has to be in the file name, otherwise a warning is shown.
    The file names can be sorted by the first parameter value of each file.

    Parameters
    ----------
    files : string, list of strings
    parameters : string, list of strings
    unique : boolean
        If set only one file per scan parameter value is used.
    sort : boolean

    Returns
    -------
    collections.OrderedDict

    '''
    logging.debug('Get the parameter ' + str(parameters) + ' values from ' + str(len(files)) + ' files')
    files_dict = collections.OrderedDict()
    if isinstance(files, basestring):
        files = (files, )
    if isinstance(parameters, basestring):
        parameters = (parameters, )
    parameter_values_from_file_names_dict = get_parameter_value_from_file_names(files, parameters, unique=unique, sort=sort)  # get the parameter from the file name
    for file_name in files:
        with tb.open_file(file_name, mode="r") as in_file_h5:  # open the actual file
            scan_parameter_values = collections.OrderedDict()
            try:
                scan_parameters = in_file_h5.root.scan_parameters[:]  # get the scan parameters from the scan parameter table
                if parameters is None:
                    parameters = get_scan_parameter_names(scan_parameters)
                for parameter in parameters:
                    try:
                        scan_parameter_values[parameter] = np.unique(scan_parameters[parameter]).tolist()  # different scan parameter values used
                    except ValueError:  # the scan parameter does not exists
                        pass
            except tb.NoSuchNodeError:  # scan parameter table does not exist
                try:
                    scan_parameters = get_scan_parameter(in_file_h5.root.meta_data[:])  # get the scan parameters from the meta data
                    if scan_parameters:
                        try:
                            scan_parameter_values = np.unique(scan_parameters[parameters]).tolist()  # different scan parameter values used
                        except ValueError:  # the scan parameter does not exists
                            pass
                except tb.NoSuchNodeError:  # meta data table does not exist
                    pass
            if not scan_parameter_values:  # if no scan parameter values could be set from file take the parameter found in the file name
                try:
                    scan_parameter_values = parameter_values_from_file_names_dict[file_name]
                except KeyError:  # no scan parameter found at all, neither in the file name nor in the file
                    scan_parameter_values = None
            else:  # use the parameter given in the file and cross check if it matches the file name parameter if these is given
                try:
                    for key, value in scan_parameter_values.items():
                        if value and value[0] != parameter_values_from_file_names_dict[file_name][key][0]:  # parameter value exists: check if the first value is the file name value
                            logging.warning('Parameter values in the file name and in the file differ. Take ' + str(key) + ' parameters ' + str(value) + ' found in %s.', file_name)
                except KeyError:  # parameter does not exists in the file name
                    pass
                except IndexError:
                    raise IncompleteInputError('Something wrong check!')
            if unique and scan_parameter_values is not None:
                existing = False
                for parameter in scan_parameter_values:  # loop to determine if any value of any scan parameter exists already
                    all_par_values = [values[parameter] for values in files_dict.values()]
                    if any(x in [scan_parameter_values[parameter]] for x in all_par_values):
                        existing = True
                        break
                if not existing:
                    files_dict[file_name] = scan_parameter_values
                else:
                    logging.warning('Scan parameter value(s) from %s exists already, do not add to result', file_name)
            else:
                files_dict[file_name] = scan_parameter_values
    return collections.OrderedDict(sorted(files_dict.iteritems(), key=itemgetter(1)) if sort else files_dict)

Response: Takes a list of files searches for the parameter name in the first dimension and the corresponding parameter values in the second dimension.


Instruction:
Write a Python 3 script to
check if the parameter names of all files are similar. Takes the dictionary from get_parameter_from_files output as input.

Response:
def check_parameter_similarity(files_dict):
    """
    Checks if the parameter names of all files are similar. Takes the dictionary from get_parameter_from_files output as input.

    """
    try:
        parameter_names = files_dict.itervalues().next().keys()  # get the parameter names of the first file, to check if these are the same in the other files
    except AttributeError:  # if there is no parameter at all
        if any(i is not None for i in files_dict.itervalues()):  # check if there is also no parameter for the other files
            return False
        else:
            return True
    if any(parameter_names != i.keys() for i in files_dict.itervalues()):
        return False
    return True


Instruction:
Implement a Python 3 function for
taking a dict of hdf5 files and combines their meta data tables into one numpy record array.

Response:
def combine_meta_data(files_dict, meta_data_v2=True):
    """
    Takes the dict of hdf5 files and combines their meta data tables into one new numpy record array.

    Parameters
    ----------
    meta_data_v2 : bool
        True for new (v2) meta data format, False for the old (v1) format.
    """
    if len(files_dict) > 10:
        logging.info("Combine the meta data from %d files", len(files_dict))
    # determine total length needed for the new combined array, thats the fastest way to combine arrays
    total_length = 0  # the total length of the new table
    for file_name in files_dict.iterkeys():
        with tb.open_file(file_name, mode="r") as in_file_h5:  # open the actual file
            total_length += in_file_h5.root.meta_data.shape[0]

    if meta_data_v2:
        meta_data_combined = np.empty((total_length, ), dtype=[
            ('index_start', np.uint32),
            ('index_stop', np.uint32),
            ('data_length', np.uint32),
            ('timestamp_start', np.float64),
            ('timestamp_stop', np.float64),
            ('error', np.uint32)])
    else:
        meta_data_combined = np.empty((total_length, ), dtype=[
            ('start_index', np.uint32),
            ('stop_index', np.uint32),
            ('length', np.uint32),
            ('timestamp', np.float64),
            ('error', np.uint32)])

    if len(files_dict) > 10:
        progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=total_length, term_width=80)
        progress_bar.start()

    index = 0

    # fill actual result array
    for file_name in files_dict.iterkeys():
        with tb.open_file(file_name, mode="r") as in_file_h5:  # open the actual file
            array_length = in_file_h5.root.meta_data.shape[0]
            meta_data_combined[index:index + array_length] = in_file_h5.root.meta_data[:]
            index += array_length
            if len(files_dict) > 10:
                progress_bar.update(index)
    if len(files_dict) > 10:
        progress_bar.finish()
    return meta_data_combined


Instruction:
Create a Python 3 function for
returning the dy / dx of a spline curve with the fit and differentiation of a spline curve.

Response:
def smooth_differentiation(x, y, weigths=None, order=5, smoothness=3, derivation=1):
    '''Returns the dy/dx(x) with the fit and differentiation of a spline curve

    Parameters
    ----------
    x : array like
    y : array like

    Returns
    -------
    dy/dx : array like
    '''
    if (len(x) != len(y)):
        raise ValueError("x, y must have the same length")
    f = splrep(x, y, w=weigths, k=order, s=smoothness)  # spline function
    return splev(x, f, der=derivation)


Instruction:
Explain what the following Python 3 code does
def reduce_sorted_to_intersect(ar1, ar2):
    # Ravel both arrays, behavior for the first array could be different
    ar1 = np.asarray(ar1).ravel()
    ar2 = np.asarray(ar2).ravel()

    # get min max values of the arrays
    ar1_biggest_value = ar1[-1]
    ar1_smallest_value = ar1[0]
    ar2_biggest_value = ar2[-1]
    ar2_smallest_value = ar2[0]

    if ar1_biggest_value < ar2_smallest_value or ar1_smallest_value > ar2_biggest_value:  # special case, no intersection at all
        return ar1[0:0], ar2[0:0]

    # get min/max indices with values that are also in the other array
    min_index_ar1 = np.argmin(ar1 < ar2_smallest_value)
    max_index_ar1 = np.argmax(ar1 > ar2_biggest_value)
    min_index_ar2 = np.argmin(ar2 < ar1_smallest_value)
    max_index_ar2 = np.argmax(ar2 > ar1_biggest_value)

    if min_index_ar1 < 0:
        min_index_ar1 = 0
    if min_index_ar2 < 0:
        min_index_ar2 = 0
    if max_index_ar1 == 0 or max_index_ar1 > ar1.shape[0]:
        max_index_ar1 = ar1.shape[0]
    if max_index_ar2 == 0 or max_index_ar2 > ar2.shape[0]:
        max_index_ar2 = ar2.shape[0]

    # reduce the data
    return ar1[min_index_ar1:max_index_ar1], ar2[min_index_ar2:max_index_ar2]

Response: Takes two sorted arrays and return the intersection ar1 in ar2 in ar1.


Instruction:
Write a Python 3 script to
return the values that appear at least twice in array.

Response:
def get_not_unique_values(array):
    '''Returns the values that appear at least twice in array.

    Parameters
    ----------
    array : array like

    Returns
    -------
    numpy.array
    '''
    s = np.sort(array, axis=None)
    s = s[s[1:] == s[:-1]]
    return np.unique(s)


Instruction:
Explain what the following Python 3 code does
def get_meta_data_index_at_scan_parameter(meta_data_array, scan_parameter_name):
    '''Takes the analyzed meta_data table and returns the indices where the scan parameter changes

    Parameters
    ----------
    meta_data_array : numpy.recordarray
    scan_parameter_name : string

    Returns
    -------
    numpy.ndarray:
        first dimension: scan parameter value
        second dimension: index where scan parameter value was used first
    '''
    scan_parameter_values = meta_data_array[scan_parameter_name]
    diff = np.concatenate(([1], np.diff(scan_parameter_values)))
    idx = np.concatenate((np.where(diff)[0], [len(scan_parameter_values)]))
    index = np.empty(len(idx) - 1, dtype={'names': [scan_parameter_name, 'index'], 'formats': ['u4', 'u4']})
    index[scan_parameter_name] = scan_parameter_values[idx[:-1]]
    index['index'] = idx[:-1]
    return index

Response: Takes the analyzed meta_data table and returns the indices where the scan parameter changes


Instruction:
How would you explain what the following Python 3 function does
def select_hits(hits_array, condition=None):
    '''Selects the hits with condition.
    E.g.: condition = 'rel_BCID == 7 & event_number < 1000'

    Parameters
    ----------
    hits_array : numpy.array
    condition : string
        A condition that is applied to the hits in numexpr. Only if the expression evaluates to True the hit is taken.

    Returns
    -------
    numpy.array
        hit array with the selceted hits
    '''
    if condition is None:
        return hits_array

    for variable in set(re.findall(r'[a-zA-Z_]+', condition)):
        exec(variable + ' = hits_array[\'' + variable + '\']')

    return hits_array[ne.evaluate(condition)]

Response: Selects the hits with condition.


Instruction:
Write a Python 3 function that can
select the hits that occurred in events and optional selection criterion.

Response:
def get_hits_in_events(hits_array, events, assume_sorted=True, condition=None):
    '''Selects the hits that occurred in events and optional selection criterion.
        If a event range can be defined use the get_data_in_event_range function. It is much faster.

    Parameters
    ----------
    hits_array : numpy.array
    events : array
    assume_sorted : bool
        Is true if the events to select are sorted from low to high value. Increases speed by 35%.
    condition : string
        A condition that is applied to the hits in numexpr. Only if the expression evaluates to True the hit is taken.

    Returns
    -------
    numpy.array
        hit array with the hits in events.
    '''

    logging.debug("Calculate hits that exists in the given %d events." % len(events))
    if assume_sorted:
        events, _ = reduce_sorted_to_intersect(events, hits_array['event_number'])  # reduce the event number range to the max min event number of the given hits to save time
        if events.shape[0] == 0:  # if there is not a single selected hit
            return hits_array[0:0]
    try:
        if assume_sorted:
            selection = analysis_utils.in1d_events(hits_array['event_number'], events)
        else:
            logging.warning('Events are usually sorted. Are you sure you want this?')
            selection = np.in1d(hits_array['event_number'], events)
        if condition is None:
            hits_in_events = hits_array[selection]
        else:
            # bad hack to be able to use numexpr
            for variable in set(re.findall(r'[a-zA-Z_]+', condition)):
                exec(variable + ' = hits_array[\'' + variable + '\']')

            hits_in_events = hits_array[ne.evaluate(condition + ' & selection')]
    except MemoryError:
        logging.error('There are too many hits to do in RAM operations. Consider decreasing chunk size and use the write_hits_in_events function instead.')
        raise MemoryError
    return hits_in_events


Instruction:
How would you explain what the following Python 3 function does
def get_hits_of_scan_parameter(input_file_hits, scan_parameters=None, try_speedup=False, chunk_size=10000000):
    '''Takes the hit table of a hdf5 file and returns hits in chunks for each unique combination of scan_parameters.
    Yields the hits in chunks, since they usually do not fit into memory.

    Parameters
    ----------
    input_file_hits : pytable hdf5 file
        Has to include a hits node
    scan_parameters : iterable with strings
    try_speedup : bool
        If true a speed up by searching for the event numbers in the data is done. If the event numbers are not in the data
        this slows down the search.
    chunk_size : int
        How many rows of data are read into ram.

    Returns
    -------
    Yields tuple, numpy.array
        Actual scan parameter tuple, hit array with the hits of a chunk of the given scan parameter tuple
    '''

    with tb.open_file(input_file_hits, mode="r+") as in_file_h5:
        hit_table = in_file_h5.root.Hits
        meta_data = in_file_h5.root.meta_data[:]
        meta_data_table_at_scan_parameter = get_unique_scan_parameter_combinations(meta_data, scan_parameters=scan_parameters)
        parameter_values = get_scan_parameters_table_from_meta_data(meta_data_table_at_scan_parameter, scan_parameters)
        event_number_ranges = get_ranges_from_array(meta_data_table_at_scan_parameter['event_number'])  # get the event number ranges for the different scan parameter settings
        index_event_number(hit_table)  # create a event_numer index to select the hits by their event number fast, no needed but important for speed up
#
        # variables for read speed up
        index = 0  # index where to start the read out of the hit table, 0 at the beginning, increased during looping
        best_chunk_size = chunk_size  # number of hits to copy to RAM during looping, the optimal chunk size is determined during looping

        # loop over the selected events
        for parameter_index, (start_event_number, stop_event_number) in enumerate(event_number_ranges):
            logging.debug('Read hits for ' + str(scan_parameters) + ' = ' + str(parameter_values[parameter_index]))

            readout_hit_len = 0  # variable to calculate a optimal chunk size value from the number of hits for speed up
            # loop over the hits in the actual selected events with optimizations: determine best chunk size, start word index given
            for hits, index in data_aligned_at_events(hit_table, start_event_number=start_event_number, stop_event_number=stop_event_number, start_index=index, try_speedup=try_speedup, chunk_size=best_chunk_size):
                yield parameter_values[parameter_index], hits
                readout_hit_len += hits.shape[0]
            best_chunk_size = int(1.5 * readout_hit_len) if int(1.05 * readout_hit_len) < chunk_size and int(1.05 * readout_hit_len) > 1e3 else chunk_size

Response: Takes the hit table of a hdf5 file and returns hits in chunks for each unique combination of scan_parameters.


Instruction:
How would you explain what the following Python 3 function does
def get_data_in_event_range(array, event_start=None, event_stop=None, assume_sorted=True):
    '''Selects the data (rows of a table) that occurred in the given event range [event_start, event_stop[

    Parameters
    ----------
    array : numpy.array
    event_start : int, None
    event_stop : int, None
    assume_sorted : bool
        Set to true if the hits are sorted by the event_number. Increases speed.

    Returns
    -------
    numpy.array
        hit array with the hits in the event range.
    '''
    logging.debug("Calculate data of the the given event range [" + str(event_start) + ", " + str(event_stop) + "[")
    event_number = array['event_number']
    if assume_sorted:
        data_event_start = event_number[0]
        data_event_stop = event_number[-1]
        if (event_start is not None and event_stop is not None) and (data_event_stop < event_start or data_event_start > event_stop or event_start == event_stop):  # special case, no intersection at all
            return array[0:0]

        # get min/max indices with values that are also in the other array
        if event_start is None:
            min_index_data = 0
        else:
            min_index_data = np.argmin(event_number < event_start)

        if event_stop is None:
            max_index_data = event_number.shape[0]
        else:
            max_index_data = np.argmax(event_number >= event_stop)

        if min_index_data < 0:
            min_index_data = 0
        if max_index_data == 0 or max_index_data > event_number.shape[0]:
            max_index_data = event_number.shape[0]
        return array[min_index_data:max_index_data]
    else:
        return array[ne.evaluate('event_number >= event_start & event_number < event_stop')]

Response: Selects the data that occurred in the given event range [ event_start event_stop ]


Instruction:
How would you implement a function in Python 3 that
selects the hits that occurred in events and writes them to a pytable.

Response:
def write_hits_in_events(hit_table_in, hit_table_out, events, start_hit_word=0, chunk_size=5000000, condition=None):
    '''Selects the hits that occurred in events and writes them to a pytable. This function reduces the in RAM operations and has to be
    used if the get_hits_in_events function raises a memory error. Also a condition can be set to select hits.

    Parameters
    ----------
    hit_table_in : pytable.table
    hit_table_out : pytable.table
        functions need to be able to write to hit_table_out
    events : array like
        defines the events to be written from hit_table_in to hit_table_out. They do not have to exists at all.
    start_hit_word: int
        Index of the first hit word to be analyzed. Used for speed up.
    chunk_size : int
        defines how many hits are analyzed in RAM. Bigger numbers increase the speed, too big numbers let the program crash with a memory error.
    condition : string
        A condition that is applied to the hits in numexpr style. Only if the expression evaluates to True the hit is taken.

    Returns
    -------
    start_hit_word: int
        Index of the last hit word analyzed. Used to speed up the next call of write_hits_in_events.
    '''
    if len(events) > 0:  # needed to avoid crash
        min_event = np.amin(events)
        max_event = np.amax(events)
        logging.debug("Write hits from hit number >= %d that exists in the selected %d events with %d <= event number <= %d into a new hit table." % (start_hit_word, len(events), min_event, max_event))
        table_size = hit_table_in.shape[0]
        iHit = 0
        for iHit in range(start_hit_word, table_size, chunk_size):
            hits = hit_table_in.read(iHit, iHit + chunk_size)
            last_event_number = hits[-1]['event_number']
            hit_table_out.append(get_hits_in_events(hits, events=events, condition=condition))
            if last_event_number > max_event:  # speed up, use the fact that the hits are sorted by event_number
                return iHit
    return start_hit_word


Instruction:
Can you tell what is the following Python 3 function doing
def write_hits_in_event_range(hit_table_in, hit_table_out, event_start=None, event_stop=None, start_hit_word=0, chunk_size=5000000, condition=None):
    '''Selects the hits that occurred in given event range [event_start, event_stop[ and write them to a pytable. This function reduces the in RAM
       operations and has to be used if the get_data_in_event_range function raises a memory error. Also a condition can be set to select hits.

    Parameters
    ----------
    hit_table_in : pytable.table
    hit_table_out : pytable.table
        functions need to be able to write to hit_table_out
    event_start, event_stop : int, None
        start/stop event numbers. Stop event number is excluded. If None start/stop is set automatically.
    chunk_size : int
        defines how many hits are analyzed in RAM. Bigger numbers increase the speed, too big numbers let the program crash with a memory error.
    condition : string
        A condition that is applied to the hits in numexpr style. Only if the expression evaluates to True the hit is taken.
    Returns
    -------
    start_hit_word: int
        Index of the last hit word analyzed. Used to speed up the next call of write_hits_in_events.
    '''

    logging.debug('Write hits that exists in the given event range from + ' + str(event_start) + ' to ' + str(event_stop) + ' into a new hit table')
    table_size = hit_table_in.shape[0]
    for iHit in range(0, table_size, chunk_size):
        hits = hit_table_in.read(iHit, iHit + chunk_size)
        last_event_number = hits[-1]['event_number']
        selected_hits = get_data_in_event_range(hits, event_start=event_start, event_stop=event_stop)
        if condition is not None:
            # bad hack to be able to use numexpr
            for variable in set(re.findall(r'[a-zA-Z_]+', condition)):
                exec(variable + ' = hits[\'' + variable + '\']')
            selected_hits = selected_hits[ne.evaluate(condition)]
        hit_table_out.append(selected_hits)
        if last_event_number > event_stop:  # speed up, use the fact that the hits are sorted by event_number
            return iHit + chunk_size
    return start_hit_word

Response: This function writes hits that occurred in given event range to a hit table.


Instruction:
Given the following Python 3 function, write the documentation
def get_events_with_n_cluster(event_number, condition='n_cluster==1'):
    '''Selects the events with a certain number of cluster.

    Parameters
    ----------
    event_number : numpy.array

    Returns
    -------
    numpy.array
    '''

    logging.debug("Calculate events with clusters where " + condition)
    n_cluster_in_events = analysis_utils.get_n_cluster_in_events(event_number)
    n_cluster = n_cluster_in_events[:, 1]
#    return np.take(n_cluster_in_events, ne.evaluate(condition), axis=0)  # does not return 1d, bug?
    return n_cluster_in_events[ne.evaluate(condition), 0]

Response: Selects the events with a certain number of cluster.


Instruction:
Write a Python 3 script to
select the events with cluster of a given cluster size.

Response:
def get_events_with_cluster_size(event_number, cluster_size, condition='cluster_size==1'):
    '''Selects the events with cluster of a given cluster size.

    Parameters
    ----------
    event_number : numpy.array
    cluster_size : numpy.array
    condition : string

    Returns
    -------
    numpy.array
    '''

    logging.debug("Calculate events with clusters with " + condition)
    return np.unique(event_number[ne.evaluate(condition)])


Instruction:
Can you create a Python 3 function that
selects the events with a certain error code.

Response:
def get_events_with_error_code(event_number, event_status, select_mask=0b1111111111111111, condition=0b0000000000000000):
    '''Selects the events with a certain error code.

    Parameters
    ----------
    event_number : numpy.array
    event_status : numpy.array
    select_mask : int
        The mask that selects the event error code to check.
    condition : int
        The value the selected event error code should have.

    Returns
    -------
    numpy.array
    '''

    logging.debug("Calculate events with certain error code")
    return np.unique(event_number[event_status & select_mask == condition])


Instruction:
Write a Python 3 function for
taking the numpy meta data array and returns the different scan parameter settings and the name aligned in a dictionary

Response:
def get_scan_parameter(meta_data_array, unique=True):
    '''Takes the numpy meta data array and returns the different scan parameter settings and the name aligned in a dictionary

    Parameters
    ----------
    meta_data_array : numpy.ndarray
    unique: boolean
        If true only unique values for each scan parameter are returned

    Returns
    -------
    python.dict{string, numpy.Histogram}:
        A dictionary with the scan parameter name/values pairs
    '''

    try:
        last_not_parameter_column = meta_data_array.dtype.names.index('error_code')  # for interpreted meta_data
    except ValueError:
        last_not_parameter_column = meta_data_array.dtype.names.index('error')  # for raw data file meta_data
    if last_not_parameter_column == len(meta_data_array.dtype.names) - 1:  # no meta_data found
        return
    scan_parameters = collections.OrderedDict()
    for scan_par_name in meta_data_array.dtype.names[4:]:  # scan parameters are in columns 5 (= index 4) and above
        scan_parameters[scan_par_name] = np.unique(meta_data_array[scan_par_name]) if unique else meta_data_array[scan_par_name]
    return scan_parameters


Instruction:
Explain what the following Python 3 code does
def get_scan_parameters_table_from_meta_data(meta_data_array, scan_parameters=None):
    '''Takes the meta data array and returns the scan parameter values as a view of a numpy array only containing the parameter data .
    Parameters
    ----------
    meta_data_array : numpy.ndarray
        The array with the scan parameters.
    scan_parameters : list of strings
        The name of the scan parameters to take. If none all are used.

    Returns
    -------
    numpy.Histogram
    '''

    if scan_parameters is None:
        try:
            last_not_parameter_column = meta_data_array.dtype.names.index('error_code')  # for interpreted meta_data
        except ValueError:
            return
        if last_not_parameter_column == len(meta_data_array.dtype.names) - 1:  # no meta_data found
            return
        # http://stackoverflow.com/questions/15182381/how-to-return-a-view-of-several-columns-in-numpy-structured-array
        scan_par_data = {name: meta_data_array.dtype.fields[name] for name in meta_data_array.dtype.names[last_not_parameter_column + 1:]}
    else:
        scan_par_data = collections.OrderedDict()
        for name in scan_parameters:
            scan_par_data[name] = meta_data_array.dtype.fields[name]

    return np.ndarray(meta_data_array.shape, np.dtype(scan_par_data), meta_data_array, 0, meta_data_array.strides)

Response: Takes the meta data array and returns the scan parameter values as a view of a numpy array only containing the parameter data.


Instruction:
Write a Python 3 function that can
take the scan parameter array and creates a scan parameter index labeling the unique scan parameter combinations.

Response:
def get_scan_parameters_index(scan_parameter):
    '''Takes the scan parameter array and creates a scan parameter index labeling the unique scan parameter combinations.
    Parameters
    ----------
    scan_parameter : numpy.ndarray
        The table with the scan parameters.

    Returns
    -------
    numpy.Histogram
    '''
    _, index = np.unique(scan_parameter, return_index=True)
    index = np.sort(index)
    values = np.array(range(0, len(index)), dtype='i4')
    index = np.append(index, len(scan_parameter))
    counts = np.diff(index)
    return np.repeat(values, counts)


Instruction:
Explain what the following Python 3 code does
def get_unique_scan_parameter_combinations(meta_data_array, scan_parameters=None, scan_parameter_columns_only=False):
    '''Takes the numpy meta data array and returns the first rows with unique combinations of different scan parameter values for selected scan parameters.
        If selected columns only is true, the returned histogram only contains the selected columns.

    Parameters
    ----------
    meta_data_array : numpy.ndarray
    scan_parameters : list of string, None
        Scan parameter names taken. If None all are used.
    selected_columns_only : bool

    Returns
    -------
    numpy.Histogram
    '''

    try:
        last_not_parameter_column = meta_data_array.dtype.names.index('error_code')  # for interpreted meta_data
    except ValueError:
        last_not_parameter_column = meta_data_array.dtype.names.index('error')  # for raw data file meta_data
    if last_not_parameter_column == len(meta_data_array.dtype.names) - 1:  # no meta_data found
        return
    if scan_parameters is None:
        return unique_row(meta_data_array, use_columns=range(4, len(meta_data_array.dtype.names)), selected_columns_only=scan_parameter_columns_only)
    else:
        use_columns = []
        for scan_parameter in scan_parameters:
            try:
                use_columns.append(meta_data_array.dtype.names.index(scan_parameter))
            except ValueError:
                logging.error('No scan parameter ' + scan_parameter + ' found')
                raise RuntimeError('Scan parameter not found')
        return unique_row(meta_data_array, use_columns=use_columns, selected_columns_only=scan_parameter_columns_only)

Response: Takes the numpy meta data array and returns the first rows with unique combinations of different scan parameter values for selected scan parameters.


Instruction:
Can you tell what is the following Python 3 function doing
def data_aligned_at_events(table, start_event_number=None, stop_event_number=None, start_index=None, stop_index=None, chunk_size=10000000, try_speedup=False, first_event_aligned=True, fail_on_missing_events=True):
    '''Takes the table with a event_number column and returns chunks with the size up to chunk_size. The chunks are chosen in a way that the events are not splitted.
    Additional parameters can be set to increase the readout speed. Events between a certain range can be selected.
    Also the start and the stop indices limiting the table size can be specified to improve performance.
    The event_number column must be sorted.
    In case of try_speedup is True, it is important to create an index of event_number column with pytables before using this function. Otherwise the queries are slowed down.

    Parameters
    ----------
    table : pytables.table
        The data.
    start_event_number : int
        The retruned data contains events with event number >= start_event_number. If None, no limit is set.
    stop_event_number : int
        The retruned data contains events with event number < stop_event_number. If None, no limit is set.
    start_index : int
        Start index of data. If None, no limit is set.
    stop_index : int
        Stop index of data. If None, no limit is set.
    chunk_size : int
        Maximum chunk size per read.
    try_speedup : bool
        If True, try to reduce the index range to read by searching for the indices of start and stop event number. If these event numbers are usually
        not in the data this speedup can even slow down the function!

    The following parameters are not used when try_speedup is True:

    first_event_aligned : bool
        If True, assuming that the first event is aligned to the data chunk and will be added. If False, the lowest event number of the first chunk will not be read out.
    fail_on_missing_events : bool
        If True, an error is given when start_event_number or stop_event_number is not part of the data.

    Returns
    -------
    Iterator of tuples
        Data of the actual data chunk and start index for the next chunk.

    Example
    -------
    start_index = 0
    for scan_parameter in scan_parameter_range:
        start_event_number, stop_event_number = event_select_function(scan_parameter)
        for data, start_index in data_aligned_at_events(table, start_event_number=start_event_number, stop_event_number=stop_event_number, start_index=start_index):
            do_something(data)

    for data, index in data_aligned_at_events(table):
        do_something(data)
    '''
    # initialize variables
    start_index_known = False
    stop_index_known = False
    start_index = 0 if start_index is None else start_index
    stop_index = table.nrows if stop_index is None else stop_index
    if stop_index < start_index:
        raise InvalidInputError('Invalid start/stop index')
    table_max_rows = table.nrows
    if stop_event_number is not None and start_event_number is not None and stop_event_number < start_event_number:
        raise InvalidInputError('Invalid start/stop event number')

    # set start stop indices from the event numbers for fast read if possible; not possible if the given event number does not exist in the data stream
    if try_speedup and table.colindexed["event_number"]:
        if start_event_number is not None:
            start_condition = 'event_number==' + str(start_event_number)
            start_indices = table.get_where_list(start_condition, start=start_index, stop=stop_index)
            if start_indices.shape[0] != 0:  # set start index if possible
                start_index = start_indices[0]
                start_index_known = True

        if stop_event_number is not None:
            stop_condition = 'event_number==' + str(stop_event_number)
            stop_indices = table.get_where_list(stop_condition, start=start_index, stop=stop_index)
            if stop_indices.shape[0] != 0:  # set the stop index if possible, stop index is excluded
                stop_index = stop_indices[0]
                stop_index_known = True

    if start_index_known and stop_index_known and start_index + chunk_size >= stop_index:  # special case, one read is enough, data not bigger than one chunk and the indices are known
        yield table.read(start=start_index, stop=stop_index), stop_index
    else:  # read data in chunks, chunks do not divide events, abort if stop_event_number is reached

        # search for begin
        current_start_index = start_index
        if start_event_number is not None:
            while current_start_index < stop_index:
                current_stop_index = min(current_start_index + chunk_size, stop_index)
                array_chunk = table.read(start=current_start_index, stop=current_stop_index)  # stop index is exclusive, so add 1
                last_event_in_chunk = array_chunk["event_number"][-1]

                if last_event_in_chunk < start_event_number:
                    current_start_index = current_start_index + chunk_size  # not there yet, continue to next read (assuming sorted events)
                else:
                    first_event_in_chunk = array_chunk["event_number"][0]
#                     if stop_event_number is not None and first_event_in_chunk >= stop_event_number and start_index != 0 and start_index == current_start_index:
#                         raise InvalidInputError('The stop event %d is missing. Change stop_event_number.' % stop_event_number)
                    if array_chunk.shape[0] == chunk_size and first_event_in_chunk == last_event_in_chunk:
                        raise InvalidInputError('Chunk size too small. Increase chunk size to fit full event.')

                    if not first_event_aligned and first_event_in_chunk == start_event_number and start_index != 0 and start_index == current_start_index:  # first event in first chunk not aligned at index 0, so take next event
                        if fail_on_missing_events:
                            raise InvalidInputError('The start event %d is missing. Change start_event_number.' % start_event_number)
                        chunk_start_index = np.searchsorted(array_chunk["event_number"], start_event_number + 1, side='left')
                    elif fail_on_missing_events and first_event_in_chunk > start_event_number and start_index == current_start_index:
                        raise InvalidInputError('The start event %d is missing. Change start_event_number.' % start_event_number)
                    elif first_event_aligned and first_event_in_chunk == start_event_number and start_index == current_start_index:
                        chunk_start_index = 0
                    else:
                        chunk_start_index = np.searchsorted(array_chunk["event_number"], start_event_number, side='left')
                        if fail_on_missing_events and array_chunk["event_number"][chunk_start_index] != start_event_number and start_index == current_start_index:
                            raise InvalidInputError('The start event %d is missing. Change start_event_number.' % start_event_number)
#                     if fail_on_missing_events and ((start_index == current_start_index and chunk_start_index == 0 and start_index != 0 and not first_event_aligned) or array_chunk["event_number"][chunk_start_index] != start_event_number):
#                         raise InvalidInputError('The start event %d is missing. Change start_event_number.' % start_event_number)
                    current_start_index = current_start_index + chunk_start_index  # calculate index for next loop
                    break
        elif not first_event_aligned and start_index != 0:
            while current_start_index < stop_index:
                current_stop_index = min(current_start_index + chunk_size, stop_index)
                array_chunk = table.read(start=current_start_index, stop=current_stop_index)  # stop index is exclusive, so add 1
                first_event_in_chunk = array_chunk["event_number"][0]
                last_event_in_chunk = array_chunk["event_number"][-1]

                if array_chunk.shape[0] == chunk_size and first_event_in_chunk == last_event_in_chunk:
                    raise InvalidInputError('Chunk size too small. Increase chunk size to fit full event.')

                chunk_start_index = np.searchsorted(array_chunk["event_number"], first_event_in_chunk + 1, side='left')
                current_start_index = current_start_index + chunk_start_index
                if not first_event_in_chunk == last_event_in_chunk:
                    break

        # data loop
        while current_start_index < stop_index:
            current_stop_index = min(current_start_index + chunk_size, stop_index)
            array_chunk = table.read(start=current_start_index, stop=current_stop_index)  # stop index is exclusive, so add 1
            first_event_in_chunk = array_chunk["event_number"][0]
            last_event_in_chunk = array_chunk["event_number"][-1]

            chunk_start_index = 0

            if stop_event_number is None:
                if current_stop_index == table_max_rows:
                    chunk_stop_index = array_chunk.shape[0]
                else:
                    chunk_stop_index = np.searchsorted(array_chunk["event_number"], last_event_in_chunk, side='left')
            else:
                if last_event_in_chunk >= stop_event_number:
                    chunk_stop_index = np.searchsorted(array_chunk["event_number"], stop_event_number, side='left')
                elif current_stop_index == table_max_rows:  # this will also add the last event of the table
                    chunk_stop_index = array_chunk.shape[0]
                else:
                    chunk_stop_index = np.searchsorted(array_chunk["event_number"], last_event_in_chunk, side='left')

            nrows = chunk_stop_index - chunk_start_index
            if nrows == 0:
                if array_chunk.shape[0] == chunk_size and first_event_in_chunk == last_event_in_chunk:
                    raise InvalidInputError('Chunk size too small to fit event. Data corruption possible. Increase chunk size to read full event.')
                elif chunk_start_index == 0:  # not increasing current_start_index
                    return
                elif stop_event_number is not None and last_event_in_chunk >= stop_event_number:
                    return
            else:
                yield array_chunk[chunk_start_index:chunk_stop_index], current_start_index + nrows + chunk_start_index

            current_start_index = current_start_index + nrows + chunk_start_index

Response: Takes the table with a event_number column and returns chunks with the size up to chunk_size.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def select_good_pixel_region(hits, col_span, row_span, min_cut_threshold=0.2, max_cut_threshold=2.0):
    '''Takes the hit array and masks all pixels with a certain occupancy.

    Parameters
    ----------
    hits : array like
        If dim > 2 the additional dimensions are summed up.
    min_cut_threshold : float
        A number to specify the minimum threshold, which pixel to take. Pixels are masked if
        occupancy < min_cut_threshold * np.ma.median(occupancy)
        0 means that no pixels are masked
    max_cut_threshold : float
        A number to specify the maximum threshold, which pixel to take. Pixels are masked if
        occupancy > max_cut_threshold * np.ma.median(occupancy)
        Can be set to None that no pixels are masked by max_cut_threshold

    Returns
    -------
    numpy.ma.array, shape=(80,336)
        The hits array with masked pixels.
    '''
    hits = np.sum(hits, axis=(-1)).astype('u8')
    mask = np.ones(shape=(80, 336), dtype=np.uint8)

    mask[min(col_span):max(col_span) + 1, min(row_span):max(row_span) + 1] = 0

    ma = np.ma.masked_where(mask, hits)
    if max_cut_threshold is not None:
        return np.ma.masked_where(np.logical_or(ma < min_cut_threshold * np.ma.median(ma), ma > max_cut_threshold * np.ma.median(ma)), ma)
    else:
        return np.ma.masked_where(ma < min_cut_threshold * np.ma.median(ma), ma)

Response: Takes the hit array and masks all pixels with a certain occupancy.


Instruction:
Create a Python 3 function to
calculate a correction factor for a single hit cluster at the given GDACs from the cluster_size_histogram via cubic interpolation.

Response:
def get_hit_rate_correction(gdacs, calibration_gdacs, cluster_size_histogram):
    '''Calculates a correction factor for single hit clusters at the given GDACs from the cluster_size_histogram via cubic interpolation.

    Parameters
    ----------
    gdacs : array like
        The GDAC settings where the threshold should be determined from the calibration
    calibration_gdacs : array like
        GDAC settings used during the source scan for the cluster size calibration.
    cluster_size_histogram : numpy.array, shape=(80,336,# of GDACs during calibration)
        The calibration array

    Returns
    -------
    numpy.array, shape=(80,336,# of GDACs during calibration)
        The threshold values for each pixel at gdacs.
    '''
    logging.info('Calculate the correction factor for the single hit cluster rate at %d given GDAC settings', len(gdacs))
    if len(calibration_gdacs) != cluster_size_histogram.shape[0]:
        raise ValueError('Length of the provided pixel GDACs does not match the dimension of the cluster size array')
    hist_sum = np.sum(cluster_size_histogram, axis=1)
    hist_rel = cluster_size_histogram / hist_sum[:, np.newaxis].astype('f4') * 100.
    maximum_rate = np.amax(hist_rel[:, 1])
    correction_factor = maximum_rate / hist_rel[:, 1]
    # sort arrays since interpolate does not work otherwise
    calibration_gdacs_sorted = np.array(calibration_gdacs)
    correction_factor_sorted = correction_factor[np.argsort(calibration_gdacs_sorted)]
    calibration_gdacs_sorted = np.sort(calibration_gdacs_sorted)
    interpolation = interp1d(calibration_gdacs_sorted.tolist(), correction_factor_sorted.tolist(), kind='cubic', bounds_error=True)
    return interpolation(gdacs)


Instruction:
Implement a Python 3 function for
calculating the mean threshold from the threshold calibration at the given gdac settings.

Response:
def get_mean_threshold_from_calibration(gdac, mean_threshold_calibration):
    '''Calculates the mean threshold from the threshold calibration at the given gdac settings. If the given gdac value was not used during caluibration
    the value is determined by interpolation.

    Parameters
    ----------
    gdacs : array like
        The GDAC settings where the threshold should be determined from the calibration
    mean_threshold_calibration : pytable
        The table created during the calibration scan.

    Returns
    -------
    numpy.array, shape=(len(gdac), )
        The mean threshold values at each value in gdacs.
    '''
    interpolation = interp1d(mean_threshold_calibration['parameter_value'], mean_threshold_calibration['mean_threshold'], kind='slinear', bounds_error=True)
    return interpolation(gdac)


Instruction:
Make a summary of the following Python 3 code
def get_pixel_thresholds_from_calibration_array(gdacs, calibration_gdacs, threshold_calibration_array, bounds_error=True):
    '''Calculates the threshold for all pixels in threshold_calibration_array at the given GDAC settings via linear interpolation. The GDAC settings used during calibration have to be given.

    Parameters
    ----------
    gdacs : array like
        The GDAC settings where the threshold should be determined from the calibration
    calibration_gdacs : array like
        GDAC settings used during calibration, needed to translate the index of the calibration array to a value.
    threshold_calibration_array : numpy.array, shape=(80,336,# of GDACs during calibration)
        The calibration array

    Returns
    -------
    numpy.array, shape=(80,336,# gdacs given)
        The threshold values for each pixel at gdacs.
    '''
    if len(calibration_gdacs) != threshold_calibration_array.shape[2]:
        raise ValueError('Length of the provided pixel GDACs does not match the third dimension of the calibration array')
    interpolation = interp1d(x=calibration_gdacs, y=threshold_calibration_array, kind='slinear', bounds_error=bounds_error)
    return interpolation(gdacs)

Response: Calculates the threshold for all pixels in threshold_calibration_array at the given GDACs.


Instruction:
Can you tell what is the following Python 3 function doing
def get_n_cluster_per_event_hist(cluster_table):
    '''Calculates the number of cluster in every event.

    Parameters
    ----------
    cluster_table : pytables.table

    Returns
    -------
    numpy.Histogram
    '''
    logging.info("Histogram number of cluster per event")
    cluster_in_events = analysis_utils.get_n_cluster_in_events(cluster_table)[:, 1]  # get the number of cluster for every event
    return np.histogram(cluster_in_events, bins=range(0, np.max(cluster_in_events) + 2))

Response: Calculates the number of cluster in every event.


Instruction:
Write a Python 3 script for
finding contiguous True regions of the array condition. Returns a 2D array where first column is the start index and second column is the end index.

Response:
def contiguous_regions(condition):
    """Finds contiguous True regions of the boolean array "condition". Returns
    a 2D array where the first column is the start index of the region and the
    second column is the end index.
    http://stackoverflow.com/questions/4494404/find-large-number-of-consecutive-values-fulfilling-condition-in-a-numpy-array
    """
    # Find the indicies of changes in "condition"
    d = np.diff(condition, n=1)
    idx, = d.nonzero()

    # We need to start things after the change in "condition". Therefore,
    # we'll shift the index by 1 to the right.
    idx += 1

    if condition[0]:
        # If the start of condition is True prepend a 0
        idx = np.r_[0, idx]

    if condition[-1]:
        # If the end of condition is True, append the length of the array
        idx = np.r_[idx, condition.size]

    # Reshape the result into two columns
    idx.shape = (-1, 2)
    return idx


Instruction:
How would you code a function in Python 3 to
check FEI4 raw data array for corrupted data.

Response:
def check_bad_data(raw_data, prepend_data_headers=None, trig_count=None):
    """Checking FEI4 raw data array for corrupted data.
    """
    consecutive_triggers = 16 if trig_count == 0 else trig_count
    is_fe_data_header = logical_and(is_fe_word, is_data_header)
    trigger_idx = np.where(is_trigger_word(raw_data) >= 1)[0]
    fe_dh_idx = np.where(is_fe_data_header(raw_data) >= 1)[0]
    n_triggers = trigger_idx.shape[0]
    n_dh = fe_dh_idx.shape[0]

    # get index of the last trigger
    if n_triggers:
        last_event_data_headers_cnt = np.where(fe_dh_idx > trigger_idx[-1])[0].shape[0]
        if consecutive_triggers and last_event_data_headers_cnt == consecutive_triggers:
            if not np.all(trigger_idx[-1] > fe_dh_idx):
                trigger_idx = np.r_[trigger_idx, raw_data.shape]
            last_event_data_headers_cnt = None
        elif last_event_data_headers_cnt != 0:
            fe_dh_idx = fe_dh_idx[:-last_event_data_headers_cnt]
        elif not np.all(trigger_idx[-1] > fe_dh_idx):
            trigger_idx = np.r_[trigger_idx, raw_data.shape]
    # if any data header, add trigger for histogramming, next readout has to have trigger word
    elif n_dh:
        trigger_idx = np.r_[trigger_idx, raw_data.shape]
        last_event_data_headers_cnt = None
    # no trigger, no data header
    # assuming correct data, return input values
    else:
        return False, prepend_data_headers, n_triggers, n_dh

#     # no triggers, check for the right amount of data headers
#     if consecutive_triggers and prepend_data_headers and prepend_data_headers + n_dh != consecutive_triggers:
#         return True, n_dh, n_triggers, n_dh

    n_triggers_cleaned = trigger_idx.shape[0]
    n_dh_cleaned = fe_dh_idx.shape[0]

    # check that trigger comes before data header
    if prepend_data_headers is None and n_triggers_cleaned and n_dh_cleaned and not trigger_idx[0] < fe_dh_idx[0]:
        return True, last_event_data_headers_cnt, n_triggers, n_dh  # FIXME: 0?
    # check that no trigger comes before the first data header
    elif consecutive_triggers and prepend_data_headers is not None and n_triggers_cleaned and n_dh_cleaned and trigger_idx[0] < fe_dh_idx[0]:
        return True, last_event_data_headers_cnt, n_triggers, n_dh  # FIXME: 0?
    # check for two consecutive triggers
    elif consecutive_triggers is None and prepend_data_headers == 0 and n_triggers_cleaned and n_dh_cleaned and trigger_idx[0] < fe_dh_idx[0]:
        return True, last_event_data_headers_cnt, n_triggers, n_dh  # FIXME: 0?
    elif prepend_data_headers is not None:
        trigger_idx += (prepend_data_headers + 1)
        fe_dh_idx += (prepend_data_headers + 1)
        # for histogramming add trigger at index 0
        trigger_idx = np.r_[0, trigger_idx]
        fe_dh_idx = np.r_[range(1, prepend_data_headers + 1), fe_dh_idx]

    event_hist, bins = np.histogram(fe_dh_idx, trigger_idx)
    if consecutive_triggers is None and np.any(event_hist == 0):
        return True, last_event_data_headers_cnt, n_triggers, n_dh
    elif consecutive_triggers and np.any(event_hist != consecutive_triggers):
        return True, last_event_data_headers_cnt, n_triggers, n_dh

    return False, last_event_data_headers_cnt, n_triggers, n_dh


Instruction:
How would you explain what the following Python 3 function does
def consecutive(data, stepsize=1):
    return np.split(data, np.where(np.diff(data) != stepsize)[0] + 1)

Response: Converts array into chunks with consecutive elements of given step size.


Instruction:
Can you tell what is the following Python 3 function doing
def print_raw_data_file(input_file, start_index=0, limit=200, flavor='fei4b', select=None, tdc_trig_dist=False, trigger_data_mode=0, meta_data_v2=True):
    with tb.open_file(input_file + '.h5', mode="r") as file_h5:
        if meta_data_v2:
            index_start = file_h5.root.meta_data.read(field='index_start')
            index_stop = file_h5.root.meta_data.read(field='index_stop')
        else:
            index_start = file_h5.root.meta_data.read(field='start_index')
            index_stop = file_h5.root.meta_data.read(field='stop_index')
        total_words = 0
        for read_out_index, (index_start, index_stop) in enumerate(np.column_stack((index_start, index_stop))):
            if start_index < index_stop:
                print "\nchunk %d with length %d (from index %d to %d)\n" % (read_out_index, (index_stop - index_start), index_start, index_stop)
                raw_data = file_h5.root.raw_data.read(index_start, index_stop)
                total_words += print_raw_data(raw_data=raw_data, start_index=max(start_index - index_start, 0), limit=limit - total_words, flavor=flavor, index_offset=index_start, select=select, tdc_trig_dist=tdc_trig_dist, trigger_data_mode=trigger_data_mode)
                if limit and total_words >= limit:
                    break

Response: Print FEI4 raw data from a file.


Instruction:
How would you explain what the following Python 3 function does
def print_raw_data(raw_data, start_index=0, limit=200, flavor='fei4b', index_offset=0, select=None, tdc_trig_dist=False, trigger_data_mode=0):
    if not select:
        select = ['DH', 'TW', "AR", "VR", "SR", "DR", 'TDC', 'UNKNOWN FE WORD', 'UNKNOWN WORD']
    total_words = 0
    for index in range(start_index, raw_data.shape[0]):
        dw = FEI4Record(raw_data[index], chip_flavor=flavor, tdc_trig_dist=tdc_trig_dist, trigger_data_mode=trigger_data_mode)
        if dw in select:
            print index + index_offset, '{0:12d} {1:08b} {2:08b} {3:08b} {4:08b}'.format(raw_data[index], (raw_data[index] & 0xFF000000) >> 24, (raw_data[index] & 0x00FF0000) >> 16, (raw_data[index] & 0x0000FF00) >> 8, (raw_data[index] & 0x000000FF) >> 0), dw
            total_words += 1
            if limit and total_words >= limit:
                break
    return total_words

Response: Print FEI4 raw data array for debugging.


Instruction:
Explain what the following Python 3 code does
def update(self, pbar):
        'Updates the widget to show the ETA or total time when finished.'
        self.n_refresh += 1
        if pbar.currval == 0:
            return 'ETA:  --:--:--'
        elif pbar.finished:
            return 'Time: %s' % self.format_time(pbar.seconds_elapsed)
        else:
            elapsed = pbar.seconds_elapsed
            try:
                speed = pbar.currval / elapsed
                if self.speed_smooth is not None:
                    self.speed_smooth = (self.speed_smooth * (1 - self.SMOOTHING)) + (speed * self.SMOOTHING)
                else:
                    self.speed_smooth = speed
                eta = float(pbar.maxval) / self.speed_smooth - elapsed + 1 if float(pbar.maxval) / self.speed_smooth - elapsed + 1 > 0 else 0

                if float(pbar.currval) / pbar.maxval > 0.30 or self.n_refresh > 10:  # ETA only rather precise if > 30% is already finished or more than 10 times updated
                    return 'ETA:  %s' % self.format_time(eta)
                if self.old_eta is not None and self.old_eta < eta:  # do not show jumping ETA if non precise mode is active
                    return 'ETA: ~%s' % self.format_time(self.old_eta)
                else:
                    self.old_eta = eta
                    return 'ETA: ~%s' % self.format_time(eta)
            except ZeroDivisionError:
                speed = 0

Response: Updates the widget to show the ETA or total time when finished.


Instruction:
How would you explain what the following Python 3 function does
def plot_result(x_p, y_p, y_p_e, smoothed_data, smoothed_data_diff, filename=None):
    ''' Fit spline to the profile histogramed data, differentiate, determine MPV and plot.
     Parameters
    ----------
        x_p, y_p : array like
            data points (x,y)
        y_p_e : array like
            error bars in y
    '''
    logging.info('Plot results')
    plt.close()

    p1 = plt.errorbar(x_p * analysis_configuration['vcal_calibration'], y_p, yerr=y_p_e, fmt='o')  # plot data with error bars
    p2, = plt.plot(x_p * analysis_configuration['vcal_calibration'], smoothed_data, '-r')  # plot smoothed data
    factor = np.amax(y_p) / np.amin(smoothed_data_diff) * 1.1
    p3, = plt.plot(x_p * analysis_configuration['vcal_calibration'], factor * smoothed_data_diff, '-', lw=2)  # plot differentiated data
    mpv_index = np.argmax(-analysis_utils.smooth_differentiation(x_p, y_p, weigths=1 / y_p_e, order=3, smoothness=analysis_configuration['smoothness'], derivation=1))
    p4, = plt.plot([x_p[mpv_index] * analysis_configuration['vcal_calibration'], x_p[mpv_index] * analysis_configuration['vcal_calibration']], [0, factor * smoothed_data_diff[mpv_index]], 'k-', lw=2)
    text = 'MPV ' + str(int(x_p[mpv_index] * analysis_configuration['vcal_calibration'])) + ' e'
    plt.text(1.01 * x_p[mpv_index] * analysis_configuration['vcal_calibration'], -10. * smoothed_data_diff[mpv_index], text, ha='left')
    plt.legend([p1, p2, p3, p4], ['data', 'smoothed spline', 'spline differentiation', text], prop={'size': 12}, loc=0)
    plt.title('\'Single hit cluster\'-occupancy for different pixel thresholds')
    plt.xlabel('Pixel threshold [e]')
    plt.ylabel('Single hit cluster occupancy [a.u.]')
    plt.ylim(0, np.amax(y_p) * 1.15)
    if filename is None:
        plt.show()
    else:
        filename.savefig(plt.gcf())
    return smoothed_data_diff

Response: Fit spline to the profile histogramed data and differentiate determine MPV and plot.


Instruction:
Can you implement a function in Python 3 that
generates HitOR calibration file from raw data file and plotting of calibration data.

Response:
def create_hitor_calibration(output_filename, plot_pixel_calibrations=False):
    '''Generating HitOr calibration file (_calibration.h5) from raw data file and plotting of calibration data.

    Parameters
    ----------
    output_filename : string
        Input raw data file name.
    plot_pixel_calibrations : bool, iterable
        If True, genearating additional pixel calibration plots. If list of column and row tuples (from 1 to 80 / 336), print selected pixels.

    Returns
    -------
    nothing
    '''
    logging.info('Analyze HitOR calibration data and plot results of %s', output_filename)

    with AnalyzeRawData(raw_data_file=output_filename, create_pdf=True) as analyze_raw_data:  # Interpret the raw data file
        analyze_raw_data.create_occupancy_hist = False  # too many scan parameters to do in ram histogramming
        analyze_raw_data.create_hit_table = True
        analyze_raw_data.create_tdc_hist = True
        analyze_raw_data.align_at_tdc = True  # align events at TDC words, first word of event has to be a tdc word
        analyze_raw_data.interpret_word_table()
        analyze_raw_data.interpreter.print_summary()
        analyze_raw_data.plot_histograms()

        n_injections = analyze_raw_data.n_injections  # use later
        meta_data = analyze_raw_data.out_file_h5.root.meta_data[:]
        scan_parameters_dict = get_scan_parameter(meta_data)
        inner_loop_parameter_values = scan_parameters_dict[next(reversed(scan_parameters_dict))]  # inner loop parameter name is unknown
        scan_parameter_names = scan_parameters_dict.keys()
#         col_row_combinations = get_unique_scan_parameter_combinations(analyze_raw_data.out_file_h5.root.meta_data[:], scan_parameters=('column', 'row'), scan_parameter_columns_only=True)

        meta_data_table_at_scan_parameter = get_unique_scan_parameter_combinations(meta_data, scan_parameters=scan_parameter_names)
        scan_parameter_values = get_scan_parameters_table_from_meta_data(meta_data_table_at_scan_parameter, scan_parameter_names)
        event_number_ranges = get_ranges_from_array(meta_data_table_at_scan_parameter['event_number'])
        event_ranges_per_parameter = np.column_stack((scan_parameter_values, event_number_ranges))
        if analyze_raw_data.out_file_h5.root.Hits.nrows == 0:
            raise AnalysisError("Found no hits.")
        hits = analyze_raw_data.out_file_h5.root.Hits[:]
        event_numbers = hits['event_number'].copy()  # create contigous array, otherwise np.searchsorted too slow, http://stackoverflow.com/questions/15139299/performance-of-numpy-searchsorted-is-poor-on-structured-arrays

        output_filename = os.path.splitext(output_filename)[0]
        with tb.open_file(output_filename + "_calibration.h5", mode="w") as calibration_data_file:
            logging.info('Create calibration')
            calibration_data = np.full(shape=(80, 336, len(inner_loop_parameter_values), 4), fill_value=np.nan, dtype='f4')  # result of the calibration is a histogram with col_index, row_index, plsrDAC value, mean discrete tot, rms discrete tot, mean tot from TDC, rms tot from TDC

            progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=len(event_ranges_per_parameter), term_width=80)
            progress_bar.start()

            for index, (actual_scan_parameter_values, event_start, event_stop) in enumerate(event_ranges_per_parameter):
                if event_stop is None:  # happens for the last chunk
                    event_stop = hits[-1]['event_number'] + 1
                array_index = np.searchsorted(event_numbers, np.array([event_start, event_stop]))
                actual_hits = hits[array_index[0]:array_index[1]]
                for item_index, item in enumerate(scan_parameter_names):
                    if item == "column":
                        actual_col = actual_scan_parameter_values[item_index]
                    elif item == "row":
                        actual_row = actual_scan_parameter_values[item_index]
                    elif item == "PlsrDAC":
                        plser_dac = actual_scan_parameter_values[item_index]
                    else:
                        raise ValueError("Unknown scan parameter %s" % item)

                # Only pixel of actual column/row should be in the actual data chunk but since FIFO is not cleared for each scan step due to speed reasons and there might be noisy pixels this is not always the case
                n_wrong_pixel = np.count_nonzero(np.logical_or(actual_hits['column'] != actual_col, actual_hits['row'] != actual_row))
                if n_wrong_pixel != 0:
                    logging.warning('%d hit(s) from other pixels for scan parameters %s', n_wrong_pixel, ', '.join(['%s=%s' % (name, value) for (name, value) in zip(scan_parameter_names, actual_scan_parameter_values)]))

                actual_hits = actual_hits[np.logical_and(actual_hits['column'] == actual_col, actual_hits['row'] == actual_row)]  # Only take data from selected pixel
                actual_tdc_hits = actual_hits[(actual_hits['event_status'] & 0b0000111110011100) == 0b0000000100000000]  # only take hits from good events (one TDC word only, no error)
                actual_tot_hits = actual_hits[(actual_hits['event_status'] & 0b0000100010011100) == 0b0000000000000000]  # only take hits from good events for tot
                tot, tdc = actual_tot_hits['tot'], actual_tdc_hits['TDC']

                if tdc.shape[0] < n_injections:
                    logging.info('%d of %d expected TDC hits for scan parameters %s', tdc.shape[0], n_injections, ', '.join(['%s=%s' % (name, value) for (name, value) in zip(scan_parameter_names, actual_scan_parameter_values)]))
                if tot.shape[0] < n_injections:
                    logging.info('%d of %d expected hits for scan parameters %s', tot.shape[0], n_injections, ', '.join(['%s=%s' % (name, value) for (name, value) in zip(scan_parameter_names, actual_scan_parameter_values)]))

                inner_loop_scan_parameter_index = np.where(plser_dac == inner_loop_parameter_values)[0][0]  # translate the scan parameter value to an index for the result histogram
                # numpy mean and std return nan if array is empty
                calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 0] = np.mean(tot)
                calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 1] = np.mean(tdc)
                calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 2] = np.std(tot)
                calibration_data[actual_col - 1, actual_row - 1, inner_loop_scan_parameter_index, 3] = np.std(tdc)

                progress_bar.update(index)
            progress_bar.finish()

            calibration_data_out = calibration_data_file.create_carray(calibration_data_file.root, name='HitOrCalibration', title='Hit OR calibration data', atom=tb.Atom.from_dtype(calibration_data.dtype), shape=calibration_data.shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
            calibration_data_out[:] = calibration_data
            calibration_data_out.attrs.dimensions = scan_parameter_names
            calibration_data_out.attrs.scan_parameter_values = inner_loop_parameter_values
            calibration_data_out.flush()
#             with PdfPages(output_filename + "_calibration.pdf") as output_pdf:
            plot_scurves(calibration_data[:, :, :, 0], inner_loop_parameter_values, "ToT calibration", "ToT", 15, "Charge [PlsrDAC]", filename=analyze_raw_data.output_pdf)
            plot_scurves(calibration_data[:, :, :, 1], inner_loop_parameter_values, "TDC calibration", "TDC [ns]", None, "Charge [PlsrDAC]", filename=analyze_raw_data.output_pdf)
            tot_mean_all_pix = np.nanmean(calibration_data[:, :, :, 0], axis=(0, 1))
            tot_error_all_pix = np.nanstd(calibration_data[:, :, :, 0], axis=(0, 1))
            tdc_mean_all_pix = np.nanmean(calibration_data[:, :, :, 1], axis=(0, 1))
            tdc_error_all_pix = np.nanstd(calibration_data[:, :, :, 1], axis=(0, 1))
            plot_tot_tdc_calibration(scan_parameters=inner_loop_parameter_values, tot_mean=tot_mean_all_pix, tot_error=tot_error_all_pix, tdc_mean=tdc_mean_all_pix, tdc_error=tdc_error_all_pix, filename=analyze_raw_data.output_pdf, title="Mean charge calibration of %d pixel(s)" % np.count_nonzero(~np.all(np.isnan(calibration_data[:, :, :, 0]), axis=2)))
            # plotting individual pixels
            if plot_pixel_calibrations is True:
                # selecting pixels with non-nan entries
                col_row_non_nan = np.nonzero(~np.all(np.isnan(calibration_data[:, :, :, 0]), axis=2))
                plot_pixel_calibrations = np.dstack(col_row_non_nan)[0]
            elif plot_pixel_calibrations is False:
                plot_pixel_calibrations = np.array([], dtype=np.int)
            else:  # assuming list of column / row tuples
                plot_pixel_calibrations = np.array(plot_pixel_calibrations) - 1
            # generate index array
            pixel_indices = np.arange(plot_pixel_calibrations.shape[0])
            plot_n_pixels = 10  # number of pixels at the beginning, center and end of the array
            np.random.seed(0)
            # select random pixels
            if pixel_indices.size - 2 * plot_n_pixels >= 0:
                random_pixel_indices = np.sort(np.random.choice(pixel_indices[plot_n_pixels:-plot_n_pixels], min(plot_n_pixels, pixel_indices.size - 2 * plot_n_pixels), replace=False))
            else:
                random_pixel_indices = np.array([], dtype=np.int)
            selected_pixel_indices = np.unique(np.hstack([pixel_indices[:plot_n_pixels], random_pixel_indices, pixel_indices[-plot_n_pixels:]]))
            # plotting individual pixels
            for (column, row) in plot_pixel_calibrations[selected_pixel_indices]:
                logging.info("Plotting charge calibration for pixel column " + str(column + 1) + " / row " + str(row + 1))
                tot_mean_single_pix = calibration_data[column, row, :, 0]
                tot_std_single_pix = calibration_data[column, row, :, 2]
                tdc_mean_single_pix = calibration_data[column, row, :, 1]
                tdc_std_single_pix = calibration_data[column, row, :, 3]
                plot_tot_tdc_calibration(scan_parameters=inner_loop_parameter_values, tot_mean=tot_mean_single_pix, tot_error=tot_std_single_pix, tdc_mean=tdc_mean_single_pix, tdc_error=tdc_std_single_pix, filename=analyze_raw_data.output_pdf, title="Charge calibration for pixel column " + str(column + 1) + " / row " + str(row + 1))


Instruction:
Write a Python 3 function that can
send a run status mail with the traceback to a specified E - Mail address.

Response:
def send_mail(subject, body, smtp_server, user, password, from_addr, to_addrs):
    ''' Sends a run status mail with the traceback to a specified E-Mail address if a run crashes.
    '''
    logging.info('Send status E-Mail (' + subject + ')')
    content = string.join((
        "From: %s" % from_addr,
        "To: %s" % ','.join(to_addrs),  # comma separated according to RFC822
        "Subject: %s" % subject,
        "",
        body),
        "\r\n")
    server = smtplib.SMTP_SSL(smtp_server)
    server.login(user, password)
    server.sendmail(from_addr, to_addrs, content)
    server.quit()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _parse_module_cfgs(self):
        ''' Extracts the configuration of the modules.
        '''
        # Adding here default run config parameters.
        if "dut" not in self._conf or self._conf["dut"] is None:
            raise ValueError('Parameter "dut" not defined.')
        if "dut_configuration" not in self._conf or self._conf["dut_configuration"] is None:
            raise ValueError('Parameter "dut_configuration" not defined.')
        self._conf.setdefault('working_dir', None)  # string, if None, absolute path of configuration.yaml file will be used

        if 'modules' in self._conf and self._conf['modules']:
            for module_id, module_cfg in [(key, value) for key, value in self._conf['modules'].items() if ("activate" not in value or ("activate" in value and value["activate"] is True))]:
                # Check here for missing module config items.
                # Capital letter keys are Basil drivers, other keys are parameters.
                # FIFO, RX, TX, TLU and TDC are generic driver names which are used in the scan implementations.
                # The use of these reserved driver names allows for abstraction.
                # Accessing Basil drivers with real name is still possible.
                if "module_group" in module_id:
                    raise ValueError('The module ID "%s" contains the reserved name "module_group".' % module_id)
                if "flavor" not in module_cfg or module_cfg["flavor"] is None:
                    raise ValueError('No parameter "flavor" defined for module "%s".' % module_id)
                if module_cfg["flavor"] in fe_flavors:
                    for driver_name in _reserved_driver_names:
                        # TDC is not mandatory
                        if driver_name == "TDC":
                            # TDC is allowed to have set None
                            module_cfg.setdefault('TDC', None)
                            continue
                        if driver_name not in module_cfg or module_cfg[driver_name] is None:
                            raise ValueError('No parameter "%s" defined for module "%s".' % (driver_name, module_id))
                    if "rx_channel" not in module_cfg or module_cfg["rx_channel"] is None:
                        raise ValueError('No parameter "rx_channel" defined for module "%s".' % module_id)
                    if "tx_channel" not in module_cfg or module_cfg["tx_channel"] is None:
                        raise ValueError('No parameter "tx_channel" defined for module "%s".' % module_id)
                    if "chip_address" not in module_cfg:
                        raise ValueError('No parameter "chip_address" defined for module "%s".' % module_id)
                    module_cfg.setdefault("tdc_channel", None)
                    module_cfg.setdefault("configuration", None)  # string or number, if None, using the last valid configuration
                    module_cfg.setdefault("send_data", None)  # address string of PUB socket
                    module_cfg.setdefault("activate", True)  # set module active by default
                    # Save config to dict.
                    self._module_cfgs[module_id] = module_cfg
                    self._modules[module_id] = [module_id]
        else:
            raise ValueError("No module configuration specified")

Response: Extracts the configuration of the modules.


Instruction:
Can you implement a function in Python 3 that
sets the default parameters if they are not specified.

Response:
def _set_default_cfg(self):
        ''' Sets the default parameters if they are not specified.
        '''
        # adding special conf for accessing all DUT drivers
        self._module_cfgs[None] = {
            'flavor': None,
            'chip_address': None,
            'FIFO': list(set([self._module_cfgs[module_id]['FIFO'] for module_id in self._modules])),
            'RX': list(set([self._module_cfgs[module_id]['RX'] for module_id in self._modules])),
            'rx_channel': list(set([self._module_cfgs[module_id]['rx_channel'] for module_id in self._modules])),
            'TX': list(set([self._module_cfgs[module_id]['TX'] for module_id in self._modules])),
            'tx_channel': list(set([self._module_cfgs[module_id]['tx_channel'] for module_id in self._modules])),
            'TDC': list(set([self._module_cfgs[module_id]['TDC'] for module_id in self._modules])),
            'tdc_channel': list(set([self._module_cfgs[module_id]['tdc_channel'] for module_id in self._modules])),
            'TLU': list(set([self._module_cfgs[module_id]['TLU'] for module_id in self._modules])),
            'configuration': None,
            'send_data': None}

        tx_groups = groupby_dict({key: value for (key, value) in self._module_cfgs.items() if key in self._modules}, "TX")
        for tx, module_group in tx_groups.items():
            flavors = list(set([module_cfg['flavor'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group]))
            if len(flavors) != 1:
                raise ValueError("Parameter 'flavor' must be the same for module group TX=%s." % tx)

            chip_addresses = list(set([module_cfg['chip_address'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group]))
            if len(module_group) != len(chip_addresses) or (len(module_group) != 1 and None in chip_addresses):
                raise ValueError("Parameter 'chip_address' must be different for each module in module group TX=%s." % tx)

            # Adding broadcast config for parallel mode.
            self._module_cfgs["module_group_TX=" + tx] = {
                'flavor': flavors[0],
                'chip_address': None,  # broadcast
                'FIFO': list(set([module_cfg['FIFO'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'RX': list(set([module_cfg['RX'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'rx_channel': list(set([module_cfg['rx_channel'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'TX': tx,
                'tx_channel': list(set([module_cfg['tx_channel'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'TDC': list(set([module_cfg['TDC'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'tdc_channel': list(set([module_cfg['tdc_channel'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'TLU': list(set([module_cfg['TLU'] for module_id, module_cfg in self._module_cfgs.items() if module_id in module_group])),
                'configuration': None,
                'send_data': None}
            self._tx_module_groups["module_group_TX=" + tx] = module_group

        # Setting up per module attributes
        self._module_attr = {key: {} for key in self._module_cfgs}
        # Setting up per module run conf
        for module_id in self._module_cfgs:
            sc = namedtuple('run_configuration', field_names=self._default_run_conf.keys())
            run_conf = sc(**self._run_conf)
            if module_id in self._modules and self.__class__.__name__ in self._conf["modules"][module_id] and self._conf["modules"][module_id][self.__class__.__name__] is not None:
                self._module_run_conf[module_id] = run_conf._replace(**self._conf["modules"][module_id][self.__class__.__name__])._asdict()
            else:
                self._module_run_conf[module_id] = run_conf._asdict()
                # update module group with run specific configuration
                if module_id in self._tx_module_groups and self._tx_module_groups[module_id]:
                    selected_module_id = self._tx_module_groups[module_id][0]
                    if self.__class__.__name__ in self._conf["modules"][selected_module_id] and self._conf["modules"][selected_module_id][self.__class__.__name__] is not None:
                        self._module_run_conf[module_id] = run_conf._replace(**self._conf["modules"][selected_module_id][self.__class__.__name__])._asdict()


Instruction:
How would you explain what the following Python 3 function does
def init_modules(self):
        ''' Initialize all modules consecutively'''
        for module_id, module_cfg in self._module_cfgs.items():
            if module_id in self._modules or module_id in self._tx_module_groups:
                if module_id in self._modules:
                    module_id_str = "module " + module_id
                else:
                    module_id_str = module_id.split('=', 1)
                    module_id_str[0] = module_id_str[0].replace("_", " ")
                    module_id_str = "=".join(module_id_str)
                logging.info("Initializing configuration for %s..." % module_id_str)
                # adding scan parameters to dict
                if 'scan_parameters' in self._module_run_conf[module_id] and self._module_run_conf[module_id]['scan_parameters'] is not None:
                    # evaluating string for support of nested lists and other complex data structures
                    if isinstance(self._module_run_conf[module_id]['scan_parameters'], basestring):
                        self._module_run_conf[module_id]['scan_parameters'] = ast.literal_eval(self._module_run_conf[module_id]['scan_parameters'])
                    sp = namedtuple('scan_parameters', field_names=zip(*self._module_run_conf[module_id]['scan_parameters'])[0])
                    self._scan_parameters[module_id] = sp(*zip(*self._module_run_conf[module_id]['scan_parameters'])[1])
                else:
                    sp = namedtuple_with_defaults('scan_parameters', field_names=[])
                    self._scan_parameters[module_id] = sp()
                # init FE config
                if module_id in self._modules:
                    # only real modules can have an existing configuration
                    last_configuration = self.get_configuration(module_id=module_id)
                else:
                    last_configuration = None
                if (('configuration' not in module_cfg or module_cfg['configuration'] is None) and last_configuration is None) or (isinstance(module_cfg['configuration'], (int, long)) and module_cfg['configuration'] <= 0):
                    if 'chip_address' in module_cfg:
                        if module_cfg['chip_address'] is None:
                            chip_address = 0
                            broadcast = True
                        else:
                            chip_address = module_cfg['chip_address']
                            broadcast = False
                    else:
                        raise ValueError('Parameter "chip_address" not specified for module "%s".' % module_id)
                    if 'flavor' in module_cfg and module_cfg['flavor']:
                        module_cfg['configuration'] = FEI4Register(fe_type=module_cfg['flavor'], chip_address=chip_address, broadcast=broadcast)
                    else:
                        raise ValueError('Parameter "flavor" not specified for module "%s".' % module_id)
                # use existing config
                elif not module_cfg['configuration'] and last_configuration:
                    module_cfg['configuration'] = FEI4Register(configuration_file=last_configuration)
                # path string
                elif isinstance(module_cfg['configuration'], basestring):
                    if os.path.isabs(module_cfg['configuration']):  # absolute path
                        module_cfg['configuration'] = FEI4Register(configuration_file=module_cfg['configuration'])
                    else:  # relative path
                        module_cfg['configuration'] = FEI4Register(configuration_file=os.path.join(module_cfg['working_dir'], module_cfg['configuration']))
                # run number
                elif isinstance(module_cfg['configuration'], (int, long)) and module_cfg['configuration'] > 0:
                    module_cfg['configuration'] = FEI4Register(configuration_file=self.get_configuration(module_id=module_id, run_number=module_cfg['configuration']))
                # assume configuration already initialized
                elif not isinstance(module_cfg['configuration'], FEI4Register):
                    raise ValueError('Found no valid value for parameter "configuration" for module "%s".' % module_id)

                # init register utils
                self._registers[module_id] = self._module_cfgs[module_id]['configuration']
                self._register_utils[module_id] = FEI4RegisterUtils(self._module_dut[module_id], self._module_cfgs[module_id]['configuration'])

                if module_id in self._modules:
                    # Create module data path for real modules
                    module_path = self.get_module_path(module_id)
                    if not os.path.exists(module_path):
                        os.makedirs(module_path)

        # Set all modules to conf mode to prevent from receiving BCR and ECR broadcast
        for module_id in self._tx_module_groups:
            with self.access_module(module_id=module_id):
                self.register_utils.set_conf_mode()

        # Initial configuration (reset and configuration) of all modules.
        # This is done by iterating over each module individually
        for module_id in self._modules:
            logging.info("Configuring %s..." % module_id)
            with self.access_module(module_id=module_id):
                if self._run_conf['configure_fe']:
                    self.register_utils.global_reset()
                    self.register_utils.configure_all()
                else:
                    self.register_utils.set_conf_mode()
                if is_fe_ready(self):
                    fe_not_ready = False
                else:
                    fe_not_ready = True
                # BCR and ECR might result in RX errors
                # a reset of the RX and FIFO will happen just before scan()
                if self._run_conf['reset_fe']:
                    self.register_utils.reset_bunch_counter()
                    self.register_utils.reset_event_counter()
                if fe_not_ready:
                    # resetting service records must be done once after power up
                    self.register_utils.reset_service_records()
                    if not is_fe_ready(self):
                        logging.warning('Module "%s" is not sending any data.' % module_id)
                # set all modules to conf mode afterwards to be immune to ECR and BCR
                self.register_utils.set_conf_mode()

Response: Initialize all modules consecutively


Instruction:
Make a summary of the following Python 3 code
def do_run(self):
        ''' Start runs on all modules sequentially.

        Sets properties to access current module properties.
        '''
        if self.broadcast_commands:  # Broadcast FE commands
            if self.threaded_scan:
                with ExitStack() as restore_config_stack:
                    # Configure each FE individually
                    # Sort module config keys, configure broadcast modules first
                    for module_id in itertools.chain(self._tx_module_groups, self._modules):
                        if self.abort_run.is_set():
                            break
                        with self.access_module(module_id=module_id):
                            if module_id in self._modules:
                                module_id_str = "module " + module_id
                            else:
                                module_id_str = module_id.split('=', 1)
                                module_id_str[0] = module_id_str[0].replace("_", " ")
                                module_id_str = "=".join(module_id_str)
                            logging.info('Scan parameter(s) for %s: %s', module_id_str, ', '.join(['%s=%s' % (key, value) for (key, value) in self.scan_parameters._asdict().items()]) if self.scan_parameters else 'None')
                            # storing register values until scan has finished and then restore configuration
                            restore_config_stack.enter_context(self.register.restored(name=self.run_number))
                            self.configure()
                    for module_id in self._tx_module_groups:
                        if self.abort_run.is_set():
                            break
                        with self.access_module(module_id=module_id):
                            # set all modules to run mode by before entering scan()
                            self.register_utils.set_run_mode()

                    with self.access_module(module_id=None):
                        self.fifo_readout.reset_rx()
                        self.fifo_readout.reset_fifo(self._selected_fifos)
                        self.fifo_readout.print_fei4_rx_status()

                        with self.access_files():
                            self._scan_threads = []
                            for module_id in self._tx_module_groups:
                                if self.abort_run.is_set():
                                    break
                                t = ExcThread(target=self.scan, name=module_id)
                                t.daemon = True  # exiting program even when thread is alive
                                self._scan_threads.append(t)
                            for t in self._scan_threads:
                                t.start()
                            while any([t.is_alive() for t in self._scan_threads]):
#                                 if self.abort_run.is_set():
#                                     break
                                for t in self._scan_threads:
                                    try:
                                        t.join(0.01)
                                    except Exception:
                                        self._scan_threads.remove(t)
                                        self.handle_err(sys.exc_info())
#                             alive_threads = [t.name for t in self._scan_threads if (not t.join(10.0) and t.is_alive())]
#                             if alive_threads:
#                                 raise RuntimeError("Scan thread(s) not finished: %s" % ", ".join(alive_threads))
                            self._scan_threads = []
                for module_id in self._tx_module_groups:
                    if self.abort_run.is_set():
                        break
                    with self.access_module(module_id=module_id):
                        # set modules to conf mode by after finishing scan()
                        self.register_utils.set_conf_mode()
            else:
                for tx_module_id, tx_group in self._tx_module_groups.items():
                    if self.abort_run.is_set():
                        break
                    with ExitStack() as restore_config_stack:
                        for module_id in itertools.chain([tx_module_id], tx_group):
                            if self.abort_run.is_set():
                                break
                            with self.access_module(module_id=module_id):
                                logging.info('Scan parameter(s) for module %s: %s', module_id, ', '.join(['%s=%s' % (key, value) for (key, value) in self.scan_parameters._asdict().items()]) if self.scan_parameters else 'None')
                                # storing register values until scan has finished and then restore configuration
                                restore_config_stack.enter_context(self.register.restored(name=self.run_number))
                                self.configure()
                        with self.access_module(module_id=tx_module_id):
                            # set all modules to run mode by before entering scan()
                            self.register_utils.set_run_mode()

                            self.fifo_readout.reset_rx()
                            self.fifo_readout.reset_fifo(self._selected_fifos)
                            self.fifo_readout.print_fei4_rx_status()

                            # some scans use this event to stop scan loop, clear event here to make another scan possible
                            self.stop_run.clear()
                            with self.access_files():
                                self.scan()

                    with self.access_module(module_id=tx_module_id):
                        # set modules to conf mode by after finishing scan()
                        self.register_utils.set_conf_mode()
        else:  # Scan each FE individually
            if self.threaded_scan:
                self._scan_threads = []
                # loop over grpups of modules with different TX
                for tx_module_ids in zip_nofill(*self._tx_module_groups.values()):
                    if self.abort_run.is_set():
                        break
                    with ExitStack() as restore_config_stack:
                        for module_id in tx_module_ids:
                            if self.abort_run.is_set():
                                break
                            with self.access_module(module_id=module_id):
                                logging.info('Scan parameter(s) for module %s: %s', module_id, ', '.join(['%s=%s' % (key, value) for (key, value) in self.scan_parameters._asdict().items()]) if self.scan_parameters else 'None')
                                # storing register values until scan has finished and then restore configuration
                                restore_config_stack.enter_context(self.register.restored(name=self.run_number))
                                self.configure()
                                # set modules to run mode by before entering scan()
                                self.register_utils.set_run_mode()
                            t = ExcThread(target=self.scan, name=module_id)
                            t.daemon = True  # exiting program even when thread is alive
                            self._scan_threads.append(t)
                        with self.access_module(module_id=tx_module_ids):
                            self.fifo_readout.reset_rx()
                            self.fifo_readout.reset_fifo(self._selected_fifos)
                            self.fifo_readout.print_fei4_rx_status()

                            with self.access_files():
                                # some scans use this event to stop scan loop, clear event here to make another scan possible
                                self.stop_run.clear()
                                for t in self._scan_threads:
                                    t.start()
                                while any([t.is_alive() for t in self._scan_threads]):
#                                     if self.abort_run.is_set():
#                                         break
                                    for t in self._scan_threads:
                                        try:
                                            t.join(0.01)
                                        except Exception:
                                            self._scan_threads.remove(t)
                                            self.handle_err(sys.exc_info())
#                                 alive_threads = [t.name for t in self._scan_threads if (not t.join(10.0) and t.is_alive())]
#                                 if alive_threads:
#                                     raise RuntimeError("Scan thread(s) not finished: %s" % ", ".join(alive_threads))
                                self._scan_threads = []

                    for module_id in tx_module_ids:
                        if self.abort_run.is_set():
                            break
                        with self.access_module(module_id=module_id):
                            # set modules to conf mode by after finishing scan()
                            self.register_utils.set_conf_mode()
            else:
                for module_id in self._modules:
                    if self.abort_run.is_set():
                        break
                    # some scans use this event to stop scan loop, clear event here to make another scan possible
                    self.stop_run.clear()
                    with self.access_module(module_id=module_id):
                        logging.info('Scan parameter(s) for module %s: %s', module_id, ', '.join(['%s=%s' % (key, value) for (key, value) in self.scan_parameters._asdict().items()]) if self.scan_parameters else 'None')
                        with self.register.restored(name=self.run_number):
                            self.configure()
                            # set modules to run mode by before entering scan()
                            self.register_utils.set_run_mode()

                            self.fifo_readout.reset_rx()
                            self.fifo_readout.reset_fifo(self._selected_fifos)
                            self.fifo_readout.print_fei4_rx_status()

                            # some scans use this event to stop scan loop, clear event here to make another scan possible
                            self.stop_run.clear()
                            with self.access_files():
                                self.scan()
                            # set modules to conf mode by after finishing scan()
                            self.register_utils.set_conf_mode()

        if self._modules:
            self.fifo_readout.print_readout_status()

Response: Start runs on all modules sequentially.


Instruction:
Here you have a function in Python 3, explain what it does
def close(self):
        '''Releasing hardware resources.
        '''
        try:
            self.dut.close()
        except Exception:
            logging.warning('Closing DUT was not successful')
        else:
            logging.debug('Closed DUT')

Response: Closes the DUT and releases hardware resources.