Instruction:
Can you generate the documentation for the following Python 3 function
def adjacent(geohash, direction):
  # Based on an MIT licensed implementation by Chris Veness from:
  #   http://www.movable-type.co.uk/scripts/geohash.html
  assert direction in 'nsew', "Invalid direction: %s"%direction
  assert geohash, "Invalid geohash: %s"%geohash
  neighbor = {
    'n': [ 'p0r21436x8zb9dcf5h7kjnmqesgutwvy', 'bc01fg45238967deuvhjyznpkmstqrwx' ],
    's': [ '14365h7k9dcfesgujnmqp0r2twvyx8zb', '238967debc01fg45kmstqrwxuvhjyznp' ],
    'e': [ 'bc01fg45238967deuvhjyznpkmstqrwx', 'p0r21436x8zb9dcf5h7kjnmqesgutwvy' ],
    'w': [ '238967debc01fg45kmstqrwxuvhjyznp', '14365h7k9dcfesgujnmqp0r2twvyx8zb' ]
  }
  border = {
    'n': [ 'prxz',     'bcfguvyz' ],
    's': [ '028b',     '0145hjnp' ],
    'e': [ 'bcfguvyz', 'prxz'     ],
    'w': [ '0145hjnp', '028b'     ]
  }
  last = geohash[-1]
  parent = geohash[0:-1]
  t = len(geohash) % 2
  # Check for edge cases
  if (last in border[direction][t]) and (parent):
    parent = adjacent(parent, direction)
  return parent + BASESEQUENCE[neighbor[direction][t].index(last)]

Response: Return the adjacent geohash for a given direction.


Instruction:
Explain what the following Python 3 code does
def neighbors(geohash):
  return {
    'n':  adjacent(geohash, 'n'),
    'ne': adjacent(adjacent(geohash, 'n'), 'e'),
    'e':  adjacent(geohash, 'e'),
    'se': adjacent(adjacent(geohash, 's'), 'e'),
    's':  adjacent(geohash, 's'),
    'sw': adjacent(adjacent(geohash, 's'), 'w'),
    'w':  adjacent(geohash, 'w'),
    'nw': adjacent(adjacent(geohash, 'n'), 'w'),
    'c':  geohash
  }

Response: Return all neighboring geohashes.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def thunkify(thread_name=None, daemon=True, default_func=None):
    '''Make a function immediately return a function of no args which, when called,
    waits for the result, which will start being processed in another thread.
    Taken from https://wiki.python.org/moin/PythonDecoratorLibrary.
    '''
    def actual_decorator(f):
        @functools.wraps(f)
        def thunked(*args, **kwargs):
            result = [None]
            exc = [False, None]  # has exception?, exception info
#             wait_event = threading.Event()

            def worker_func():
                try:
                    func_result = f(*args, **kwargs)
                    result[0] = func_result
                except Exception:
                    exc[0] = True
                    exc[1] = sys.exc_info()
                    logging.error("%s has thrown an exception:\n%s", thread_name, traceback.format_exc())
#                 finally:
#                     wait_event.set()

            worker_thread = Thread(target=worker_func, name=thread_name if thread_name else None)
            worker_thread.daemon = daemon

            def thunk(timeout=None):
                # avoid blocking MainThread
                start_time = time()
                while True:
                    worker_thread.join(timeout=0.1)
                    if (timeout and timeout < time() - start_time) or not worker_thread.is_alive():
                        break
#                 worker_thread.join(timeout=timeout)
#                 wait_event.wait()
                if worker_thread.is_alive():
                    if default_func is None:
                        return
                    else:
                        return default_func()
                if exc[0]:
                    raise exc[1][0], exc[1][1], exc[1][2]
                return result[0]

            worker_thread.start()
#             threading.Thread(target=worker_func, name=thread_name if thread_name else None).start()
            return thunk
        return thunked
    return actual_decorator

Response: A decorator which returns a function immediately return a function of no args which will be processed in another thread.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_event_when_keyboard_interrupt(_lambda):
    '''Decorator function that sets Threading.Event() when keyboard interrupt (Ctrl+C) was raised

    Parameters
    ----------
    _lambda : function
        Lambda function that points to Threading.Event() object

    Returns
    -------
    wrapper : function

    Examples
    --------
    @set_event_when_keyboard_interrupt(lambda x: x.stop_thread_event)
    def scan(self, **kwargs):
        # some code

    Note
    ----
    Decorated functions cannot be derived.
    '''
    def wrapper(f):
        @wraps(f)
        def wrapped_f(self, *f_args, **f_kwargs):
            try:
                f(self, *f_args, **f_kwargs)
            except KeyboardInterrupt:
                _lambda(self).set()
#                 logging.info('Keyboard interrupt: setting %s' % _lambda(self).__name__)
        return wrapped_f
    return wrapper

Response: Decorator function that sets Threading. Event when keyboard interrupt was raised.


Instruction:
How would you explain what the following Python 3 function does
def run_id(self):
        '''Run name without whitespace
        '''
        s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__)
        return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

Response: Run name without whitespace


Instruction:
Can you generate the documentation for the following Python 3 function
def conf(self):
        '''Configuration (namedtuple)
        '''
        conf = namedtuple('conf', field_names=self._conf.keys())
        return conf(**self._conf)

Response: Return a named tuple containing the current configuration.


Instruction:
Write a Python 3 function that can
run configuration of the current session

Response:
def run_conf(self):
        '''Run configuration (namedtuple)
        '''
        run_conf = namedtuple('run_conf', field_names=self._run_conf.keys())
        return run_conf(**self._run_conf)


Instruction:
Write a Python 3 script to
default run configuration (namedtuple)

Response:
def default_run_conf(self):
        '''Default run configuration (namedtuple)
        '''
        default_run_conf = namedtuple('default_run_conf', field_names=self._default_run_conf.keys())
        return default_run_conf(**self._default_run_conf)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _init(self, run_conf, run_number=None):
        '''Initialization before a new run.
        '''
        self.stop_run.clear()
        self.abort_run.clear()
        self._run_status = run_status.running
        self._write_run_number(run_number)
        self._init_run_conf(run_conf)

Response: Initialize before a new run.


Instruction:
Here you have a function in Python 3, explain what it does
def connect_cancel(self, functions):
        '''Run given functions when a run is cancelled.
        '''
        self._cancel_functions = []
        for func in functions:
            if isinstance(func, basestring) and hasattr(self, func) and callable(getattr(self, func)):
                self._cancel_functions.append(getattr(self, func))
            elif callable(func):
                self._cancel_functions.append(func)
            else:
                raise ValueError("Unknown function %s" % str(func))

Response: Connect given functions when a run is cancelled.


Instruction:
Write a Python 3 script to
stop a run. Control for loops. Gentle stop and abort.

Response:
def stop(self, msg=None):
        '''Stopping a run. Control for loops. Gentle stop/abort.

        This event should provide a more gentle abort. The run should stop ASAP but the run is still considered complete.
        '''
        if not self.stop_run.is_set():
            if msg:
                logging.info('%s%s Stopping run...', msg, ('' if msg[-1] in punctuation else '.'))
            else:
                logging.info('Stopping run...')
        self.stop_run.set()


Instruction:
Can you write a function in Python 3 where it
aborts a run. Control for loops. Immediate stop and abort.

Response:
def abort(self, msg=None):
        '''Aborting a run. Control for loops. Immediate stop/abort.

        The implementation should stop a run ASAP when this event is set. The run is considered incomplete.
        '''
        if not self.abort_run.is_set():
            if msg:
                logging.error('%s%s Aborting run...', msg, ('' if msg[-1] in punctuation else '.'))
            else:
                logging.error('Aborting run...')
        self.abort_run.set()
        self.stop_run.set()


Instruction:
Write a Python 3 script for
running a run in another thread. Non - blocking.

Response:
def run_run(self, run, conf=None, run_conf=None, use_thread=False, catch_exception=True):
        '''Runs a run in another thread. Non-blocking.

        Parameters
        ----------
        run : class, object
            Run class or object.
        run_conf : str, dict, file
            Specific configuration for the run.
        use_thread : bool
            If True, run run in thread and returns blocking function.

        Returns
        -------
        If use_thread is True, returns function, which blocks until thread terminates, and which itself returns run status.
        If use_thread is False, returns run status.
        '''
        if isinstance(conf, basestring) and os.path.isfile(conf):
            logging.info('Updating configuration from file %s', os.path.abspath(conf))
        elif conf is not None:
            logging.info('Updating configuration')
        conf = self.open_conf(conf)
        self._conf.update(conf)

        if isclass(run):
            # instantiate the class
            run = run(conf=self._conf)

        local_run_conf = {}
        # general parameters from conf
        if 'run_conf' in self._conf:
            logging.info('Updating run configuration using run_conf key from configuration')
            local_run_conf.update(self._conf['run_conf'])
        # check for class name, scan specific parameters from conf
        if run.__class__.__name__ in self._conf:
            logging.info('Updating run configuration using %s key from configuration' % (run.__class__.__name__,))
            local_run_conf.update(self._conf[run.__class__.__name__])

        if isinstance(run_conf, basestring) and os.path.isfile(run_conf):
            logging.info('Updating run configuration from file %s', os.path.abspath(run_conf))
        elif run_conf is not None:
            logging.info('Updating run configuration')
        run_conf = self.open_conf(run_conf)
        # check for class name, scan specific parameters from conf
        if run.__class__.__name__ in run_conf:
            run_conf = run_conf[run.__class__.__name__]
        # run_conf parameter has highest priority, updated last
        local_run_conf.update(run_conf)

        if use_thread:
            self.current_run = run

            @thunkify(thread_name='RunThread', daemon=True, default_func=self.current_run.get_run_status)
            def run_run_in_thread():
                return run.run(run_conf=local_run_conf)

            signal.signal(signal.SIGINT, self._signal_handler)
            logging.info('Press Ctrl-C to stop run')

            return run_run_in_thread()
        else:
            self.current_run = run
            status = run.run(run_conf=local_run_conf)
            if not catch_exception and status != run_status.finished:
                raise RuntimeError('Exception occurred. Please read the log.')
            return status


Instruction:
Can you write a function in Python 3 where it
runs runs from a primlist.

Response:
def run_primlist(self, primlist, skip_remaining=False):
        '''Runs runs from a primlist.

        Parameters
        ----------
        primlist : string
            Filename of primlist.
        skip_remaining : bool
            If True, skip remaining runs, if a run does not exit with status FINISHED.

        Note
        ----
        Primlist is a text file of the following format (comment line by adding '#'):
        <module name (containing class) or class (in either case use dot notation)>; <scan parameter>=<value>; <another scan parameter>=<another value>
        '''
        runlist = self.open_primlist(primlist)
        for index, run in enumerate(runlist):
            logging.info('Progressing with run %i out of %i...', index + 1, len(runlist))
            join = self.run_run(run, use_thread=True)
            status = join()
            if skip_remaining and not status == run_status.finished:
                logging.error('Exited run %i with status %s: Skipping all remaining runs.', run.run_number, status)
                break


Instruction:
Here you have a function in Python 3, explain what it does
def analyze_beam_spot(scan_base, combine_n_readouts=1000, chunk_size=10000000, plot_occupancy_hists=False, output_pdf=None, output_file=None):
    ''' Determines the mean x and y beam spot position as a function of time. Therefore the data of a fixed number of read outs are combined ('combine_n_readouts'). The occupancy is determined
    for the given combined events and stored into a pdf file. At the end the beam x and y is plotted into a scatter plot with absolute positions in um.

     Parameters
    ----------
    scan_base: list of str
        scan base names (e.g.:  ['//data//SCC_50_fei4_self_trigger_scan_390', ]
    combine_n_readouts: int
        the number of read outs to combine (e.g. 1000)
    max_chunk_size: int
        the maximum chunk size used during read, if too big memory error occurs, if too small analysis takes longer
    output_pdf: PdfPages
        PdfPages file object, if none the plot is printed to screen
    '''
    time_stamp = []
    x = []
    y = []

    for data_file in scan_base:
        with tb.open_file(data_file + '_interpreted.h5', mode="r+") as in_hit_file_h5:
            # get data and data pointer
            meta_data_array = in_hit_file_h5.root.meta_data[:]
            hit_table = in_hit_file_h5.root.Hits

            # determine the event ranges to analyze (timestamp_start, start_event_number, stop_event_number)
            parameter_ranges = np.column_stack((analysis_utils.get_ranges_from_array(meta_data_array['timestamp_start'][::combine_n_readouts]), analysis_utils.get_ranges_from_array(meta_data_array['event_number'][::combine_n_readouts])))

            # create a event_numer index (important)
            analysis_utils.index_event_number(hit_table)

            # initialize the analysis and set settings
            analyze_data = AnalyzeRawData()
            analyze_data.create_tot_hist = False
            analyze_data.create_bcid_hist = False
            analyze_data.histogram.set_no_scan_parameter()

            # variables for read speed up
            index = 0  # index where to start the read out, 0 at the beginning, increased during looping
            best_chunk_size = chunk_size

            progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=hit_table.shape[0], term_width=80)
            progress_bar.start()

            # loop over the selected events
            for parameter_index, parameter_range in enumerate(parameter_ranges):
                logging.debug('Analyze time stamp ' + str(parameter_range[0]) + ' and data from events = [' + str(parameter_range[2]) + ',' + str(parameter_range[3]) + '[ ' + str(int(float(float(parameter_index) / float(len(parameter_ranges)) * 100.0))) + '%')
                analyze_data.reset()  # resets the data of the last analysis

                # loop over the hits in the actual selected events with optimizations: determine best chunk size, start word index given
                readout_hit_len = 0  # variable to calculate a optimal chunk size value from the number of hits for speed up
                for hits, index in analysis_utils.data_aligned_at_events(hit_table, start_event_number=parameter_range[2], stop_event_number=parameter_range[3], start_index=index, chunk_size=best_chunk_size):
                    analyze_data.analyze_hits(hits)  # analyze the selected hits in chunks
                    readout_hit_len += hits.shape[0]
                    progress_bar.update(index)
                best_chunk_size = int(1.5 * readout_hit_len) if int(1.05 * readout_hit_len) < chunk_size else chunk_size  # to increase the readout speed, estimated the number of hits for one read instruction

                # get and store results
                occupancy_array = analyze_data.histogram.get_occupancy()
                projection_x = np.sum(occupancy_array, axis=0).ravel()
                projection_y = np.sum(occupancy_array, axis=1).ravel()
                x.append(analysis_utils.get_mean_from_histogram(projection_x, bin_positions=range(0, 80)))
                y.append(analysis_utils.get_mean_from_histogram(projection_y, bin_positions=range(0, 336)))
                time_stamp.append(parameter_range[0])
                if plot_occupancy_hists:
                    plotting.plot_occupancy(occupancy_array[:, :, 0], title='Occupancy for events between ' + time.strftime('%H:%M:%S', time.localtime(parameter_range[0])) + ' and ' + time.strftime('%H:%M:%S', time.localtime(parameter_range[1])), filename=output_pdf)
            progress_bar.finish()
    plotting.plot_scatter([i * 250 for i in x], [i * 50 for i in y], title='Mean beam position', x_label='x [um]', y_label='y [um]', marker_style='-o', filename=output_pdf)
    if output_file:
        with tb.open_file(output_file, mode="a") as out_file_h5:
            rec_array = np.array(zip(time_stamp, x, y), dtype=[('time_stamp', float), ('x', float), ('y', float)])
            try:
                beam_spot_table = out_file_h5.create_table(out_file_h5.root, name='Beamspot', description=rec_array, title='Beam spot position', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
                beam_spot_table[:] = rec_array
            except tb.exceptions.NodeError:
                logging.warning(output_file + ' has already a Beamspot note, do not overwrite existing.')
    return time_stamp, x, y

Response: Analyzes the mean x and y of a beam spot.


Instruction:
Create a Python 3 function for
analyzing the number of events as a function of time.

Response:
def analyze_event_rate(scan_base, combine_n_readouts=1000, time_line_absolute=True, output_pdf=None, output_file=None):
    ''' Determines the number of events as a function of time. Therefore the data of a fixed number of read outs are combined ('combine_n_readouts'). The number of events is taken from the meta data info
    and stored into a pdf file.

    Parameters
    ----------
    scan_base: list of str
        scan base names (e.g.:  ['//data//SCC_50_fei4_self_trigger_scan_390', ]
    combine_n_readouts: int
        the number of read outs to combine (e.g. 1000)
    time_line_absolute: bool
        if true the analysis uses absolute time stamps
    output_pdf: PdfPages
        PdfPages file object, if none the plot is printed to screen
    '''
    time_stamp = []
    rate = []

    start_time_set = False

    for data_file in scan_base:
        with tb.open_file(data_file + '_interpreted.h5', mode="r") as in_file_h5:
            meta_data_array = in_file_h5.root.meta_data[:]
            parameter_ranges = np.column_stack((analysis_utils.get_ranges_from_array(meta_data_array['timestamp_start'][::combine_n_readouts]), analysis_utils.get_ranges_from_array(meta_data_array['event_number'][::combine_n_readouts])))

            if time_line_absolute:
                time_stamp.extend(parameter_ranges[:-1, 0])
            else:
                if not start_time_set:
                    start_time = parameter_ranges[0, 0]
                    start_time_set = True
                time_stamp.extend((parameter_ranges[:-1, 0] - start_time) / 60.0)
            rate.extend((parameter_ranges[:-1, 3] - parameter_ranges[:-1, 2]) / (parameter_ranges[:-1, 1] - parameter_ranges[:-1, 0]))  # d#Events / dt
    if time_line_absolute:
        plotting.plot_scatter_time(time_stamp, rate, title='Event rate [Hz]', marker_style='o', filename=output_pdf)
    else:
        plotting.plot_scatter(time_stamp, rate, title='Events per time', x_label='Progressed time [min.]', y_label='Events rate [Hz]', marker_style='o', filename=output_pdf)
    if output_file:
        with tb.open_file(output_file, mode="a") as out_file_h5:
            rec_array = np.array(zip(time_stamp, rate), dtype=[('time_stamp', float), ('rate', float)]).view(np.recarray)
            try:
                rate_table = out_file_h5.create_table(out_file_h5.root, name='Eventrate', description=rec_array, title='Event rate', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
                rate_table[:] = rec_array
            except tb.exceptions.NodeError:
                logging.warning(output_file + ' has already a Eventrate note, do not overwrite existing.')
    return time_stamp, rate


Instruction:
Given the following Python 3 function, write the documentation
def analyse_n_cluster_per_event(scan_base, include_no_cluster=False, time_line_absolute=True, combine_n_readouts=1000, chunk_size=10000000, plot_n_cluster_hists=False, output_pdf=None, output_file=None):
    ''' Determines the number of cluster per event as a function of time. Therefore the data of a fixed number of read outs are combined ('combine_n_readouts').

    Parameters
    ----------
    scan_base: list of str
        scan base names (e.g.:  ['//data//SCC_50_fei4_self_trigger_scan_390', ]
    include_no_cluster: bool
        Set to true to also consider all events without any hit.
    combine_n_readouts: int
        the number of read outs to combine (e.g. 1000)
    max_chunk_size: int
        the maximum chunk size used during read, if too big memory error occurs, if too small analysis takes longer
    output_pdf: PdfPages
        PdfPages file object, if none the plot is printed to screen
    '''

    time_stamp = []
    n_cluster = []

    start_time_set = False

    for data_file in scan_base:
        with tb.open_file(data_file + '_interpreted.h5', mode="r+") as in_cluster_file_h5:
            # get data and data pointer
            meta_data_array = in_cluster_file_h5.root.meta_data[:]
            cluster_table = in_cluster_file_h5.root.Cluster

            # determine the event ranges to analyze (timestamp_start, start_event_number, stop_event_number)
            parameter_ranges = np.column_stack((analysis_utils.get_ranges_from_array(meta_data_array['timestamp_start'][::combine_n_readouts]), analysis_utils.get_ranges_from_array(meta_data_array['event_number'][::combine_n_readouts])))

            # create a event_numer index (important for speed)
            analysis_utils.index_event_number(cluster_table)

            # initialize the analysis and set settings
            analyze_data = AnalyzeRawData()
            analyze_data.create_tot_hist = False
            analyze_data.create_bcid_hist = False

            # variables for read speed up
            index = 0  # index where to start the read out, 0 at the beginning, increased during looping
            best_chunk_size = chunk_size

            total_cluster = cluster_table.shape[0]

            progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=total_cluster, term_width=80)
            progress_bar.start()

            # loop over the selected events
            for parameter_index, parameter_range in enumerate(parameter_ranges):
                logging.debug('Analyze time stamp ' + str(parameter_range[0]) + ' and data from events = [' + str(parameter_range[2]) + ',' + str(parameter_range[3]) + '[ ' + str(int(float(float(parameter_index) / float(len(parameter_ranges)) * 100.0))) + '%')
                analyze_data.reset()  # resets the data of the last analysis

                # loop over the cluster in the actual selected events with optimizations: determine best chunk size, start word index given
                readout_cluster_len = 0  # variable to calculate a optimal chunk size value from the number of hits for speed up
                hist = None
                for clusters, index in analysis_utils.data_aligned_at_events(cluster_table, start_event_number=parameter_range[2], stop_event_number=parameter_range[3], start_index=index, chunk_size=best_chunk_size):
                    n_cluster_per_event = analysis_utils.get_n_cluster_in_events(clusters['event_number'])[:, 1]  # array with the number of cluster per event, cluster per event are at least 1
                    if hist is None:
                        hist = np.histogram(n_cluster_per_event, bins=10, range=(0, 10))[0]
                    else:
                        hist = np.add(hist, np.histogram(n_cluster_per_event, bins=10, range=(0, 10))[0])
                    if include_no_cluster and parameter_range[3] is not None:  # happend for the last readout
                        hist[0] = (parameter_range[3] - parameter_range[2]) - len(n_cluster_per_event)  # add the events without any cluster
                    readout_cluster_len += clusters.shape[0]
                    total_cluster -= len(clusters)
                    progress_bar.update(index)
                best_chunk_size = int(1.5 * readout_cluster_len) if int(1.05 * readout_cluster_len) < chunk_size else chunk_size  # to increase the readout speed, estimated the number of hits for one read instruction

                if plot_n_cluster_hists:
                    plotting.plot_1d_hist(hist, title='Number of cluster per event at ' + str(parameter_range[0]), x_axis_title='Number of cluster', y_axis_title='#', log_y=True, filename=output_pdf)
                hist = hist.astype('f4') / np.sum(hist)  # calculate fraction from total numbers

                if time_line_absolute:
                    time_stamp.append(parameter_range[0])
                else:
                    if not start_time_set:
                        start_time = parameter_ranges[0, 0]
                        start_time_set = True
                    time_stamp.append((parameter_range[0] - start_time) / 60.0)
                n_cluster.append(hist)
            progress_bar.finish()
            if total_cluster != 0:
                logging.warning('Not all clusters were selected during analysis. Analysis is therefore not exact')

    if time_line_absolute:
        plotting.plot_scatter_time(time_stamp, n_cluster, title='Number of cluster per event as a function of time', marker_style='o', filename=output_pdf, legend=('0 cluster', '1 cluster', '2 cluster', '3 cluster') if include_no_cluster else ('0 cluster not plotted', '1 cluster', '2 cluster', '3 cluster'))
    else:
        plotting.plot_scatter(time_stamp, n_cluster, title='Number of cluster per event as a function of time', x_label='time [min.]', marker_style='o', filename=output_pdf, legend=('0 cluster', '1 cluster', '2 cluster', '3 cluster') if include_no_cluster else ('0 cluster not plotted', '1 cluster', '2 cluster', '3 cluster'))
    if output_file:
        with tb.open_file(output_file, mode="a") as out_file_h5:
            cluster_array = np.array(n_cluster)
            rec_array = np.array(zip(time_stamp, cluster_array[:, 0], cluster_array[:, 1], cluster_array[:, 2], cluster_array[:, 3], cluster_array[:, 4], cluster_array[:, 5]), dtype=[('time_stamp', float), ('cluster_0', float), ('cluster_1', float), ('cluster_2', float), ('cluster_3', float), ('cluster_4', float), ('cluster_5', float)]).view(np.recarray)
            try:
                n_cluster_table = out_file_h5.create_table(out_file_h5.root, name='n_cluster', description=rec_array, title='Cluster per event', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
                n_cluster_table[:] = rec_array
            except tb.exceptions.NodeError:
                logging.warning(output_file + ' has already a Beamspot note, do not overwrite existing.')
    return time_stamp, n_cluster

Response: Analyse the number of cluster per event.


Instruction:
Can you generate the documentation for the following Python 3 function
def select_hits_from_cluster_info(input_file_hits, output_file_hits, cluster_size_condition, n_cluster_condition, chunk_size=4000000):
    ''' Takes a hit table and stores only selected hits into a new table. The selection is done on an event base and events are selected if they have a certain number of cluster or cluster size.
    To increase the analysis speed a event index for the input hit file is created first. Since a cluster hit table can be created to this way of hit selection is
    not needed anymore.

     Parameters
    ----------
    input_file_hits: str
        the input file name with hits
    output_file_hits: str
        the output file name for the hits
    cluster_size_condition: str
        the cluster size condition to select events (e.g.: 'cluster_size_condition <= 2')
    n_cluster_condition: str
        the number of cluster in a event ((e.g.: 'n_cluster_condition == 1')
    '''
    logging.info('Write hits of events from ' + str(input_file_hits) + ' with ' + cluster_size_condition + ' and ' + n_cluster_condition + ' into ' + str(output_file_hits))
    with tb.open_file(input_file_hits, mode="r+") as in_hit_file_h5:
        analysis_utils.index_event_number(in_hit_file_h5.root.Hits)
        analysis_utils.index_event_number(in_hit_file_h5.root.Cluster)
        with tb.open_file(output_file_hits, mode="w") as out_hit_file_h5:
            hit_table_out = out_hit_file_h5.create_table(out_hit_file_h5.root, name='Hits', description=data_struct.HitInfoTable, title='hit_data', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
            cluster_table = in_hit_file_h5.root.Cluster
            last_word_number = 0
            progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=cluster_table.shape[0], term_width=80)
            progress_bar.start()
            for data, index in analysis_utils.data_aligned_at_events(cluster_table, chunk_size=chunk_size):
                selected_events_1 = analysis_utils.get_events_with_cluster_size(event_number=data['event_number'], cluster_size=data['size'], condition=cluster_size_condition)  # select the events with clusters of a certain size
                selected_events_2 = analysis_utils.get_events_with_n_cluster(event_number=data['event_number'], condition=n_cluster_condition)  # select the events with a certain cluster number
                selected_events = analysis_utils.get_events_in_both_arrays(selected_events_1, selected_events_2)  # select events with both conditions above
                logging.debug('Selected ' + str(len(selected_events)) + ' events with ' + n_cluster_condition + ' and ' + cluster_size_condition)
                last_word_number = analysis_utils.write_hits_in_events(hit_table_in=in_hit_file_h5.root.Hits, hit_table_out=hit_table_out, events=selected_events, start_hit_word=last_word_number)  # write the hits of the selected events into a new table
                progress_bar.update(index)
            progress_bar.finish()
            in_hit_file_h5.root.meta_data.copy(out_hit_file_h5.root)

Response: Takes a hit table and stores only selected hits into a new table.


Instruction:
Create a Python 3 function for
selecting hits from the given file and stores only those that satisfy the given condition.

Response:
def select_hits(input_file_hits, output_file_hits, condition=None, cluster_size_condition=None, n_cluster_condition=None, chunk_size=5000000):
    ''' Takes a hit table and stores only selected hits into a new table. The selection of hits is done with a numexp string. Only if
    this expression evaluates to true the hit is taken. One can also select hits from cluster conditions. This selection is done
    on an event basis, meaning events are selected where the cluster condition is true and then hits of these events are taken.

     Parameters
    ----------
    input_file_hits: str
        the input file name with hits
    output_file_hits: str
        the output file name for the hits
    condition: str
        Numexpr string to select hits (e.g.: '(relative_BCID == 6) & (column == row)')
        All hit infos can be used (column, row, ...)
    cluster_size_condition: int
        Hit of events with the given cluster size are selected.
    n_cluster_condition: int
        Hit of events with the given cluster number are selected.
    '''
    logging.info('Write hits with ' + condition + ' into ' + str(output_file_hits))
    if cluster_size_condition is None and n_cluster_condition is None:  # no cluster cuts are done
        with tb.open_file(input_file_hits, mode="r+") as in_hit_file_h5:
            analysis_utils.index_event_number(in_hit_file_h5.root.Hits)  # create event index for faster selection
            with tb.open_file(output_file_hits, mode="w") as out_hit_file_h5:
                hit_table_out = out_hit_file_h5.create_table(out_hit_file_h5.root, name='Hits', description=data_struct.HitInfoTable, title='hit_data', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
                analysis_utils.write_hits_in_event_range(hit_table_in=in_hit_file_h5.root.Hits, hit_table_out=hit_table_out, condition=condition)  # write the hits of the selected events into a new table
                in_hit_file_h5.root.meta_data.copy(out_hit_file_h5.root)  # copy meta_data note to new file
    else:
        with tb.open_file(input_file_hits, mode="r+") as in_hit_file_h5:  # open file with hit/cluster data with r+ to be able to create index
            analysis_utils.index_event_number(in_hit_file_h5.root.Hits)  # create event index for faster selection
            analysis_utils.index_event_number(in_hit_file_h5.root.Cluster)  # create event index for faster selection
            with tb.open_file(output_file_hits, mode="w") as out_hit_file_h5:
                hit_table_out = out_hit_file_h5.create_table(out_hit_file_h5.root, name='Hits', description=data_struct.HitInfoTable, title='hit_data', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
                cluster_table = in_hit_file_h5.root.Cluster
                last_word_number = 0
                progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=cluster_table.shape[0], term_width=80)
                progress_bar.start()
                for data, index in analysis_utils.data_aligned_at_events(cluster_table, chunk_size=chunk_size):
                    if cluster_size_condition is not None:
                        selected_events = analysis_utils.get_events_with_cluster_size(event_number=data['event_number'], cluster_size=data['size'], condition='cluster_size == ' + str(cluster_size_condition))  # select the events with only 1 hit cluster
                        if n_cluster_condition is not None:
                            selected_events_2 = analysis_utils.get_events_with_n_cluster(event_number=data['event_number'], condition='n_cluster == ' + str(n_cluster_condition))  # select the events with only 1 cluster
                            selected_events = selected_events[analysis_utils.in1d_events(selected_events, selected_events_2)]  # select events with the first two conditions above
                    elif n_cluster_condition is not None:
                        selected_events = analysis_utils.get_events_with_n_cluster(event_number=data['event_number'], condition='n_cluster == ' + str(n_cluster_condition))
                    else:
                        raise RuntimeError('Cannot understand cluster selection criterion')
                    last_word_number = analysis_utils.write_hits_in_events(hit_table_in=in_hit_file_h5.root.Hits, hit_table_out=hit_table_out, events=selected_events, start_hit_word=last_word_number, condition=condition, chunk_size=chunk_size)  # write the hits of the selected events into a new table
                    progress_bar.update(index)
                progress_bar.finish()
                in_hit_file_h5.root.meta_data.copy(out_hit_file_h5.root)


Instruction:
How would you explain what the following Python 3 function does
def histogram_cluster_table(analyzed_data_file, output_file, chunk_size=10000000):
    '''Reads in the cluster info table in chunks and histograms the seed pixels into one occupancy array.
    The 3rd dimension of the occupancy array is the number of different scan parameters used

    Parameters
    ----------
    analyzed_data_file : string
        HDF5 filename of the file containing the cluster table. If a scan parameter is given in the meta data, the occupancy histogramming is done per scan parameter step.

    Returns
    -------
    occupancy_array: numpy.array with dimensions (col, row, #scan_parameter)
    '''

    with tb.open_file(analyzed_data_file, mode="r") as in_file_h5:
        with tb.open_file(output_file, mode="w") as out_file_h5:
            histogram = PyDataHistograming()
            histogram.create_occupancy_hist(True)
            scan_parameters = None
            event_number_indices = None
            scan_parameter_indices = None
            try:
                meta_data = in_file_h5.root.meta_data[:]
                scan_parameters = analysis_utils.get_unique_scan_parameter_combinations(meta_data)
                if scan_parameters is not None:
                    scan_parameter_indices = np.array(range(0, len(scan_parameters)), dtype='u4')
                    event_number_indices = np.ascontiguousarray(scan_parameters['event_number']).astype(np.uint64)
                    histogram.add_meta_event_index(event_number_indices, array_length=len(scan_parameters['event_number']))
                    histogram.add_scan_parameter(scan_parameter_indices)
                    logging.info("Add %d different scan parameter(s) for analysis", len(scan_parameters))
                else:
                    logging.info("No scan parameter data provided")
                    histogram.set_no_scan_parameter()
            except tb.exceptions.NoSuchNodeError:
                logging.info("No meta data provided, use no scan parameter")
                histogram.set_no_scan_parameter()

            logging.info('Histogram cluster seeds...')
            progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=in_file_h5.root.Cluster.shape[0], term_width=80)
            progress_bar.start()
            total_cluster = 0  # to check analysis
            for cluster, index in analysis_utils.data_aligned_at_events(in_file_h5.root.Cluster, chunk_size=chunk_size):
                total_cluster += len(cluster)
                histogram.add_cluster_seed_hits(cluster, len(cluster))
                progress_bar.update(index)
            progress_bar.finish()

            filter_table = tb.Filters(complib='blosc', complevel=5, fletcher32=False)  # compression of the written data
            occupancy_array = histogram.get_occupancy().T
            occupancy_array_table = out_file_h5.create_carray(out_file_h5.root, name='HistOcc', title='Occupancy Histogram', atom=tb.Atom.from_dtype(occupancy_array.dtype), shape=occupancy_array.shape, filters=filter_table)
            occupancy_array_table[:] = occupancy_array

            if total_cluster != np.sum(occupancy_array):
                logging.warning('Analysis shows inconsistent number of cluster used. Check needed!')
            in_file_h5.root.meta_data.copy(out_file_h5.root)

Response: Reads in the cluster info table in chunks and histograms the seed pixels into one occupancy array.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def analyze_hits_per_scan_parameter(analyze_data, scan_parameters=None, chunk_size=50000):
    '''Takes the hit table and analyzes the hits per scan parameter

    Parameters
    ----------
    analyze_data : analysis.analyze_raw_data.AnalyzeRawData object with an opened hit file (AnalyzeRawData.out_file_h5) or a
    file name with the hit data given (AnalyzeRawData._analyzed_data_file)
    scan_parameters : list of strings:
        The names of the scan parameters to use
    chunk_size : int:
        The chunk size of one hit table read. The bigger the faster. Too big causes memory errors.
    Returns
    -------
    yields the analysis.analyze_raw_data.AnalyzeRawData for each scan parameter
    '''

    if analyze_data.out_file_h5 is None or analyze_data.out_file_h5.isopen == 0:
        in_hit_file_h5 = tb.open_file(analyze_data._analyzed_data_file, 'r+')
        close_file = True
    else:
        in_hit_file_h5 = analyze_data.out_file_h5
        close_file = False

    meta_data = in_hit_file_h5.root.meta_data[:]  # get the meta data table
    try:
        hit_table = in_hit_file_h5.root.Hits  # get the hit table
    except tb.NoSuchNodeError:
        logging.error('analyze_hits_per_scan_parameter needs a hit table, but no hit table found.')
        return

    meta_data_table_at_scan_parameter = analysis_utils.get_unique_scan_parameter_combinations(meta_data, scan_parameters=scan_parameters)
    parameter_values = analysis_utils.get_scan_parameters_table_from_meta_data(meta_data_table_at_scan_parameter, scan_parameters)
    event_number_ranges = analysis_utils.get_ranges_from_array(meta_data_table_at_scan_parameter['event_number'])  # get the event number ranges for the different scan parameter settings

    analysis_utils.index_event_number(hit_table)  # create a event_numer index to select the hits by their event number fast, no needed but important for speed up

    # variables for read speed up
    index = 0  # index where to start the read out of the hit table, 0 at the beginning, increased during looping
    best_chunk_size = chunk_size  # number of hits to copy to RAM during looping, the optimal chunk size is determined during looping

    # loop over the selected events
    for parameter_index, (start_event_number, stop_event_number) in enumerate(event_number_ranges):
        logging.info('Analyze hits for ' + str(scan_parameters) + ' = ' + str(parameter_values[parameter_index]))
        analyze_data.reset()  # resets the front end data of the last analysis step but not the options
        readout_hit_len = 0  # variable to calculate a optimal chunk size value from the number of hits for speed up
        # loop over the hits in the actual selected events with optimizations: determine best chunk size, start word index given
        for hits, index in analysis_utils.data_aligned_at_events(hit_table, start_event_number=start_event_number, stop_event_number=stop_event_number, start_index=index, chunk_size=best_chunk_size):
            analyze_data.analyze_hits(hits, scan_parameter=False)  # analyze the selected hits in chunks
            readout_hit_len += hits.shape[0]
        best_chunk_size = int(1.5 * readout_hit_len) if int(1.05 * readout_hit_len) < chunk_size and int(1.05 * readout_hit_len) > 1e3 else chunk_size  # to increase the readout speed, estimated the number of hits for one read instruction
        file_name = " ".join(re.findall("[a-zA-Z0-9]+", str(scan_parameters))) + '_' + " ".join(re.findall("[a-zA-Z0-9]+", str(parameter_values[parameter_index])))
        analyze_data._create_additional_hit_data(safe_to_file=False)
        analyze_data._create_additional_cluster_data(safe_to_file=False)
        yield analyze_data, file_name

    if close_file:
        in_hit_file_h5.close()

Response: Takes the hit table and analyzes the hits per scan parameter


Instruction:
Can you tell what is the following Python 3 function doing
def interpret_data_from_tektronix(preamble, data):
    ''' Interprets raw data from Tektronix
    returns: lists of x, y values in seconds/volt'''
    # Y mode ("WFMPRE:PT_FMT"):
    # Xn = XZEro + XINcr (n - PT_Off)
    # Yn = YZEro + YMUlt (yn - YOFf)
    voltage = np.array(data, dtype=np.float)
    meta_data = preamble.split(',')[5].split(';')
    time_unit = meta_data[3][1:-1]
    XZEro = float(meta_data[5])
    XINcr = float(meta_data[4])
    PT_Off = float(meta_data[6])
    voltage_unit = meta_data[7][1:-1]
    YZEro = float(meta_data[10])
    YMUlt = float(meta_data[8])
    YOFf = float(meta_data[9])
    time = XZEro + XINcr * (np.arange(0, voltage.size) - PT_Off)
    voltage = YZEro + YMUlt * (voltage - YOFf)
    return time, voltage, time_unit, voltage_unit

Response: Interprets raw data from Tektronix and returns time voltage time_unit and voltage_unit


Instruction:
Can you create a Python 3 function that
reads the Chip S and N from the specified address.

Response:
def read_chip_sn(self):
    '''Reading Chip S/N

    Note
    ----
    Bits [MSB-LSB] | [15]       | [14-6]       | [5-0]
    Content        | reserved   | wafer number | chip number
    '''
    commands = []
    commands.extend(self.register.get_commands("ConfMode"))
    self.register_utils.send_commands(commands)
    with self.readout(fill_buffer=True, callback=None, errback=None):
        if self.register.fei4b:
            commands = []
            self.register.set_global_register_value('Efuse_Sense', 1)
            commands.extend(self.register.get_commands("WrRegister", name=['Efuse_Sense']))
            commands.extend(self.register.get_commands("GlobalPulse", Width=0))
            self.register.set_global_register_value('Efuse_Sense', 0)
            commands.extend(self.register.get_commands("WrRegister", name=['Efuse_Sense']))
            self.register_utils.send_commands(commands)
        commands = []
        self.register.set_global_register_value('Conf_AddrEnable', 1)
        commands.extend(self.register.get_commands("WrRegister", name=['Conf_AddrEnable']))
        chip_sn_address = self.register.get_global_register_attributes("addresses", name="Chip_SN")
        commands.extend(self.register.get_commands("RdRegister", addresses=chip_sn_address))
        self.register_utils.send_commands(commands)
    data = self.read_data()

    if data.shape[0] == 0:
        logging.error('Chip S/N: No data')
        return
    read_values = []
    for index, word in enumerate(np.nditer(data)):
        fei4_data_word = FEI4Record(word, self.register.chip_flavor)
        if fei4_data_word == 'AR':
            fei4_next_data_word = FEI4Record(data[index + 1], self.register.chip_flavor)
            if fei4_next_data_word == 'VR':
                read_value = fei4_next_data_word['value']
                read_values.append(read_value)

#     commands = []
#     commands.extend(self.register.get_commands("RunMode"))
#     self.register_utils.send_commands(commands)

    if len(read_values) == 0:
        logging.error('No Chip S/N was found')
    elif len(read_values) == 1:
        logging.info('Chip S/N: %d', read_values[0])
    else:
        logging.warning('Ambiguous Chip S/N: %s', read_values)


Instruction:
How would you explain what the following Python 3 function does
def read_global_register(self, name, overwrite_config=False):
    '''The function reads the global register, interprets the data and returns the register value.

    Parameters
    ----------
    name : register name
    overwrite_config : bool
        The read values overwrite the config in RAM if true.

    Returns
    -------
    register value
    '''
    self.register_utils.send_commands(self.register.get_commands("ConfMode"))

    with self.readout(fill_buffer=True, callback=None, errback=None):
        commands = []
        commands.extend(self.register.get_commands("RdRegister", name=name))
        self.register_utils.send_commands(commands)
    data = self.read_data()

    register_object = self.register.get_global_register_objects(name=[name])[0]
    value = BitLogic(register_object['addresses'] * 16)
    index = 0
    vr_count = 0
    for word in np.nditer(data):
        fei4_data_word = FEI4Record(word, self.register.chip_flavor)
        if fei4_data_word == 'AR':
            address_value = fei4_data_word['address']
            if address_value != register_object['address'] + index:
                raise Exception('Unexpected address from Address Record: read: %d, expected: %d' % (address_value, register_object['address'] + index))
        elif fei4_data_word == 'VR':
            vr_count += 1
            if vr_count >= 2:
                raise RuntimeError("Read more than 2 value records")
            read_value = BitLogic.from_value(fei4_data_word['value'], size=16)
            if register_object['register_littleendian']:
                read_value.reverse()
            value[index * 16 + 15:index * 16] = read_value
            index += 1
    value = value[register_object['bitlength'] + register_object['offset'] - 1:register_object['offset']]
    if register_object['littleendian']:
        value.reverse()
    value = value.tovalue()
    if overwrite_config:
        self.register.set_global_register(name, value)
    return value

Response: Reads the global register and returns the register value.


Instruction:
Here you have a function in Python 3, explain what it does
def read_pixel_register(self, pix_regs=None, dcs=range(40), overwrite_config=False):
    '''The function reads the pixel register, interprets the data and returns a masked numpy arrays with the data for the chosen pixel register.
    Pixels without any data are masked.

    Parameters
    ----------
    pix_regs : iterable, string
        List of pixel register to read (e.g. Enable, C_High, ...).
        If None all are read: "EnableDigInj", "Imon", "Enable", "C_High", "C_Low", "TDAC", "FDAC"
    dcs : iterable, int
        List of double columns to read.
    overwrite_config : bool
        The read values overwrite the config in RAM if true.

    Returns
    -------
    list of masked numpy.ndarrays
    '''
    if pix_regs is None:
        pix_regs = ["EnableDigInj", "Imon", "Enable", "C_High", "C_Low", "TDAC", "FDAC"]

    self.register_utils.send_commands(self.register.get_commands("ConfMode"))

    result = []
    for pix_reg in pix_regs:
        pixel_data = np.ma.masked_array(np.zeros(shape=(80, 336), dtype=np.uint32), mask=True)  # the result pixel array, only pixel with data are not masked
        for dc in dcs:
            with self.readout(fill_buffer=True, callback=None, errback=None):
                self.register_utils.send_commands(self.register.get_commands("RdFrontEnd", name=[pix_reg], dcs=[dc]))
            data = self.read_data()

            interpret_pixel_data(data, dc, pixel_data, invert=False if pix_reg == "EnableDigInj" else True)
        if overwrite_config:
            self.register.set_pixel_register(pix_reg, pixel_data.data)
        result.append(pixel_data)
    return result

Response: Reads the specified pixel register and returns a list of masked numpy arrays with the data for the chosen pixel register.


Instruction:
Here you have a function in Python 3, explain what it does
def is_fe_ready(self):
    '''Get FEI4 status of module.

    If FEI4 is not ready, resetting service records is necessary to bring the FEI4 to a defined state.

    Returns
    -------
    value : bool
        True if FEI4 is ready, False if the FEI4 was powered up recently and is not ready.
    '''
    with self.readout(fill_buffer=True, callback=None, errback=None):
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        commands.extend(self.register.get_commands("RdRegister", address=[1]))
#         commands.extend(self.register.get_commands("RunMode"))
        self.register_utils.send_commands(commands)
    data = self.read_data()

    if len(data) != 0:
        return True if FEI4Record(data[-1], self.register.chip_flavor) == 'VR' else False
    else:
        return False

Response: Get FEI4 status of module.


Instruction:
Can you tell what is the following Python 3 function doing
def invert_pixel_mask(mask):
    '''Invert pixel mask (0->1, 1(and greater)->0).

    Parameters
    ----------
    mask : array-like
        Mask.

    Returns
    -------
    inverted_mask : array-like
        Inverted Mask.
    '''
    inverted_mask = np.ones(shape=(80, 336), dtype=np.dtype('>u1'))
    inverted_mask[mask >= 1] = 0
    return inverted_mask

Response: Invert a pixel mask.


Instruction:
Write a Python 3 script for
generating a mask array for each of the specified steps.

Response:
def make_pixel_mask(steps, shift, default=0, value=1, enable_columns=None, mask=None):
    '''Generate pixel mask.

    Parameters
    ----------
    steps : int
        Number of mask steps, e.g. steps=3 (every third pixel is enabled), steps=336 (one pixel per column), steps=672 (one pixel per double column).
    shift : int
        Shift mask by given value to the bottom (towards higher row numbers). From 0 to (steps - 1).
    default : int
        Value of pixels that are not selected by the mask.
    value : int
        Value of pixels that are selected by the mask.
    enable_columns : list
        List of columns where the shift mask will be applied. List elements can range from 1 to 80.
    mask : array_like
        Additional mask. Must be convertible to an array of booleans with the same shape as mask array. True indicates a masked (i.e. invalid) data. Masked pixels will be set to default value.

    Returns
    -------
    mask_array : numpy.ndarray
        Mask array.

    Usage
    -----
    shift_mask = 'enable'
    steps = 3 # three step mask
    for mask_step in range(steps):
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        mask_array = make_pixel_mask(steps=steps, step=mask_step)
        self.register.set_pixel_register_value(shift_mask, mask_array)
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name=shift_mask))
        self.register_utils.send_commands(commands)
        # do something here
    '''
    shape = (80, 336)
    # value = np.zeros(dimension, dtype = np.uint8)
    mask_array = np.full(shape, default, dtype=np.uint8)
    # FE columns and rows are starting from 1
    if enable_columns:
        odd_columns = [odd - 1 for odd in enable_columns if odd % 2 != 0]
        even_columns = [even - 1 for even in enable_columns if even % 2 == 0]
    else:
        odd_columns = range(0, 80, 2)
        even_columns = range(1, 80, 2)
    odd_rows = np.arange(shift % steps, 336, steps)
    even_row_offset = ((steps // 2) + shift) % steps  # // integer devision
    even_rows = np.arange(even_row_offset, 336, steps)
    if odd_columns:
        odd_col_rows = itertools.product(odd_columns, odd_rows)  # get any combination of column and row, no for loop needed
        for odd_col_row in odd_col_rows:
            mask_array[odd_col_row[0], odd_col_row[1]] = value  # advanced indexing
    if even_columns:
        even_col_rows = itertools.product(even_columns, even_rows)
        for even_col_row in even_col_rows:
            mask_array[even_col_row[0], even_col_row[1]] = value
    if mask is not None:
        mask_array = np.ma.array(mask_array, mask=mask, fill_value=default)
        mask_array = mask_array.filled()
    return mask_array


Instruction:
How would you code a function in Python 3 to
generate a mask from column and row lists.

Response:
def make_pixel_mask_from_col_row(column, row, default=0, value=1):
    '''Generate mask from column and row lists

    Parameters
    ----------
    column : iterable, int
        List of colums values.
    row : iterable, int
        List of row values.
    default : int
        Value of pixels that are not selected by the mask.
    value : int
        Value of pixels that are selected by the mask.

    Returns
    -------
    mask : numpy.ndarray
    '''
    # FE columns and rows start from 1
    col_array = np.array(column) - 1
    row_array = np.array(row) - 1
    if np.any(col_array >= 80) or np.any(col_array < 0) or np.any(row_array >= 336) or np.any(row_array < 0):
        raise ValueError('Column and/or row out of range')
    shape = (80, 336)
    mask = np.full(shape, default, dtype=np.uint8)
    mask[col_array, row_array] = value  # advanced indexing
    return mask


Instruction:
How would you implement a function in Python 3 that
generates a box shaped mask from column and row lists. Takes the minimum and maximum value from each list.

Response:
def make_box_pixel_mask_from_col_row(column, row, default=0, value=1):
    '''Generate box shaped mask from column and row lists. Takes the minimum and maximum value from each list.

    Parameters
    ----------
    column : iterable, int
        List of colums values.
    row : iterable, int
        List of row values.
    default : int
        Value of pixels that are not selected by the mask.
    value : int
        Value of pixels that are selected by the mask.

    Returns
    -------
    numpy.ndarray
    '''
    # FE columns and rows start from 1
    col_array = np.array(column) - 1
    row_array = np.array(row) - 1
    if np.any(col_array >= 80) or np.any(col_array < 0) or np.any(row_array >= 336) or np.any(row_array < 0):
        raise ValueError('Column and/or row out of range')
    shape = (80, 336)
    mask = np.full(shape, default, dtype=np.uint8)
    if column and row:
        mask[col_array.min():col_array.max() + 1, row_array.min():row_array.max() + 1] = value  # advanced indexing
    return mask


Instruction:
Can you generate the documentation for the following Python 3 function
def make_xtalk_mask(mask):
    col, row = mask.nonzero()
    row_plus_one = row + 1
    del_index = np.where(row_plus_one > 335)
    row_plus_one = np.delete(row_plus_one, del_index)
    col_plus_one = np.delete(col.copy(), del_index)
    row_minus_one = row - 1
    del_index = np.where(row_minus_one > 335)
    row_minus_one = np.delete(row_minus_one, del_index)
    col_minus_one = np.delete(col.copy(), del_index)
    col = np.concatenate((col_plus_one, col_minus_one))
    row = np.concatenate((row_plus_one, row_minus_one))
    return make_pixel_mask_from_col_row(col + 1, row + 1)

Response: Generate an extended mask that is the same size as the pixel mask.


Instruction:
Create a Python 3 function for
generating a Chessboard mask for a given column and row.

Response:
def make_checkerboard_mask(column_distance, row_distance, column_offset=0, row_offset=0, default=0, value=1):
    """
    Generate chessboard/checkerboard mask.

    Parameters
    ----------
    column_distance : int
        Column distance of the enabled pixels.
    row_distance : int
        Row distance of the enabled pixels.
    column_offset : int
        Additional column offset which shifts the columns by the given amount.
    column_offset : int
        Additional row offset which shifts the rows by the given amount.

    Returns
    -------
    ndarray
        Chessboard mask.

    Example
    -------
    Input:
    column_distance : 6
    row_distance : 2

    Output:
    [[1 0 0 0 0 0 1 0 0 0 ... 0 0 0 0 1 0 0 0 0 0]
     [0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0]
     [0 0 0 1 0 0 0 0 0 1 ... 0 1 0 0 0 0 0 1 0 0]
     ...
     [0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0]
     [0 0 0 1 0 0 0 0 0 1 ... 0 1 0 0 0 0 0 1 0 0]
     [0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0]]
    """
    col_shape = (336,)
    col = np.full(col_shape, fill_value=default, dtype=np.uint8)
    col[::row_distance] = value
    shape = (80, 336)
    chessboard_mask = np.full(shape, fill_value=default, dtype=np.uint8)
    chessboard_mask[column_offset::column_distance * 2] = np.roll(col, row_offset)
    chessboard_mask[column_distance + column_offset::column_distance * 2] = np.roll(col, row_distance / 2 + row_offset)
    return chessboard_mask


Instruction:
Make a summary of the following Python 3 code
def scan_loop(self, command, repeat_command=100, use_delay=True, additional_delay=0, mask_steps=3, enable_mask_steps=None, enable_double_columns=None, same_mask_for_all_dc=False, fast_dc_loop=True, bol_function=None, eol_function=None, digital_injection=False, enable_shift_masks=None, disable_shift_masks=None, restore_shift_masks=True, mask=None, double_column_correction=False):
    '''Implementation of the scan loops (mask shifting, loop over double columns, repeatedly sending any arbitrary command).

    Parameters
    ----------
    command : BitVector
        (FEI4) command that will be sent out serially.
    repeat_command : int
        The number of repetitions command will be sent out each mask step.
    use_delay : bool
        Add additional delay to the command (append zeros). This helps to avoid FE data errors because of sending to many commands to the FE chip.
    additional_delay: int
        Additional delay to increase the command-to-command delay (in number of clock cycles / 25ns).
    mask_steps : int
        Number of mask steps (from 1 to 672).
    enable_mask_steps : list, tuple
        List of mask steps which will be applied. Default is all mask steps. From 0 to (mask-1). A value equal None or empty list will select all mask steps.
    enable_double_columns : list, tuple
        List of double columns which will be enabled during scan. Default is all double columns. From 0 to 39 (double columns counted from zero). A value equal None or empty list will select all double columns.
    same_mask_for_all_dc : bool
        Use same mask for all double columns. This will only affect all shift masks (see enable_shift_masks). Enabling this is in general a good idea since all double columns will have the same configuration and the scan speed can increased by an order of magnitude.
    fast_dc_loop : bool
        If True, optimize double column (DC) loop to save time. Note that bol_function and eol_function cannot do register operations, if True.
    bol_function : function
        Begin of loop function that will be called each time before sending command. Argument is a function pointer (without braces) or functor.
    eol_function : function
        End of loop function that will be called each time after sending command. Argument is a function pointer (without braces) or functor.
    digital_injection : bool
        Enables digital injection. C_High and C_Low will be disabled.
    enable_shift_masks : list, tuple
        List of enable pixel masks which will be shifted during scan. Mask set to 1 for selected pixels else 0. None will select "Enable", "C_High", "C_Low".
    disable_shift_masks : list, tuple
        List of disable pixel masks which will be shifted during scan. Mask set to 0 for selected pixels else 1. None will disable no mask.
    restore_shift_masks : bool
        Writing the initial (restored) FE pixel configuration into FE after finishing the scan loop.
    mask : array-like
        Additional mask. Must be convertible to an array of booleans with the same shape as mask array. True indicates a masked pixel. Masked pixels will be disabled during shifting of the enable shift masks, and enabled during shifting disable shift mask.
    double_column_correction : str, bool, list, tuple
        Enables double column PlsrDAC correction. If value is a filename (string) or list/tuple, the default PlsrDAC correction will be overwritten. First line of the file must be a Python list ([0, 0, ...])
    '''
    if not isinstance(command, bitarray):
        raise TypeError

    if enable_shift_masks is None:
        enable_shift_masks = ["Enable", "C_High", "C_Low"]

    if disable_shift_masks is None:
        disable_shift_masks = []

    # get PlsrDAC correction
    if isinstance(double_column_correction, basestring):  # from file
        with open(double_column_correction) as fp:
            plsr_dac_correction = list(literal_eval(fp.readline().strip()))
    elif isinstance(double_column_correction, (list, tuple)):  # from list/tuple
        plsr_dac_correction = list(double_column_correction)
    else:  # default
        if "C_High".lower() in map(lambda x: x.lower(), enable_shift_masks) and "C_Low".lower() in map(lambda x: x.lower(), enable_shift_masks):
            plsr_dac_correction = self.register.calibration_parameters['Pulser_Corr_C_Inj_High']
        elif "C_High".lower() in map(lambda x: x.lower(), enable_shift_masks):
            plsr_dac_correction = self.register.calibration_parameters['Pulser_Corr_C_Inj_Med']
        elif "C_Low".lower() in map(lambda x: x.lower(), enable_shift_masks):
            plsr_dac_correction = self.register.calibration_parameters['Pulser_Corr_C_Inj_Low']
    # initial PlsrDAC value for PlsrDAC correction
    initial_plsr_dac = self.register.get_global_register_value("PlsrDAC")
    # create restore point
    restore_point_name = str(self.run_number) + '_' + self.run_id + '_scan_loop'
    with self.register.restored(name=restore_point_name):
        # pre-calculate often used commands
        conf_mode_command = self.register.get_commands("ConfMode")[0]
        run_mode_command = self.register.get_commands("RunMode")[0]
        if use_delay:
            delay = self.register.get_commands("zeros", length=additional_delay + calculate_wait_cycles(mask_steps))[0]
            scan_loop_command = command + delay
        else:
            scan_loop_command = command

        def enable_columns(dc):
            if digital_injection:
                return [dc * 2 + 1, dc * 2 + 2]
            else:  # analog injection
                if dc == 0:
                    return [1]
                elif dc == 39:
                    return [78, 79, 80]
                else:
                    return [dc * 2, dc * 2 + 1]

        def write_double_columns(dc):
            if digital_injection:
                return [dc]
            else:  # analog injection
                if dc == 0:
                    return [0]
                elif dc == 39:
                    return [38, 39]
                else:
                    return [dc - 1, dc]

        def get_dc_address_command(dc):
            commands = []
            commands.append(conf_mode_command)
            self.register.set_global_register_value("Colpr_Addr", dc)
            commands.append(self.register.get_commands("WrRegister", name=["Colpr_Addr"])[0])
            if double_column_correction:
                self.register.set_global_register_value("PlsrDAC", initial_plsr_dac + int(round(plsr_dac_correction[dc])))
                commands.append(self.register.get_commands("WrRegister", name=["PlsrDAC"])[0])
            commands.append(run_mode_command)
            return self.register_utils.concatenate_commands(commands, byte_padding=True)

        if not enable_mask_steps:
            enable_mask_steps = range(mask_steps)

        if not enable_double_columns:
            enable_double_columns = range(40)

        # preparing for scan
        commands = []
        commands.append(conf_mode_command)
        if digital_injection is True:
            # check if C_High and/or C_Low is in enable_shift_mask and/or disable_shift_mask
            if "C_High".lower() in map(lambda x: x.lower(), enable_shift_masks) or "C_High".lower() in map(lambda x: x.lower(), disable_shift_masks):
                raise ValueError('C_High must not be shift mask when using digital injection')
            if "C_Low".lower() in map(lambda x: x.lower(), enable_shift_masks) or "C_Low".lower() in map(lambda x: x.lower(), disable_shift_masks):
                raise ValueError('C_Low must not be shift mask when using digital injection')
            # turn off all injection capacitors by default
            self.register.set_pixel_register_value("C_High", 0)
            self.register.set_pixel_register_value("C_Low", 0)
            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name=["C_Low", "C_High"], joint_write=True))
            self.register.set_global_register_value("DIGHITIN_SEL", 1)
    # self.register.set_global_register_value("CalEn", 1)  # for GlobalPulse instead Cal-Command
        else:
            self.register.set_global_register_value("DIGHITIN_SEL", 0)
            # setting EnableDigInj to 0 not necessary since DIGHITIN_SEL is turned off
    #             self.register.set_pixel_register_value("EnableDigInj", 0)

    # plotting registers
    #     plt.clf()
    #     plt.imshow(curr_en_mask.T, interpolation='nearest', aspect="auto")
    #     plt.pcolor(curr_en_mask.T)
    #     plt.colorbar()
    #     plt.savefig('mask_step' + str(mask_step) + '.pdf')

        commands.extend(self.register.get_commands("WrRegister", name=["DIGHITIN_SEL"]))
        self.register_utils.send_commands(commands)

        for mask_step in enable_mask_steps:
            if self.abort_run.is_set():
                break
            commands = []
            commands.append(conf_mode_command)
            if same_mask_for_all_dc:  # generate and write first mask step
                if disable_shift_masks:
                    curr_dis_mask = make_pixel_mask(steps=mask_steps, shift=mask_step, default=1, value=0, mask=mask)
                    map(lambda mask_name: self.register.set_pixel_register_value(mask_name, curr_dis_mask), disable_shift_masks)
                    commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False if mask is not None else True, name=disable_shift_masks, joint_write=True))
                if enable_shift_masks:
                    curr_en_mask = make_pixel_mask(steps=mask_steps, shift=mask_step, mask=mask)
                    map(lambda mask_name: self.register.set_pixel_register_value(mask_name, curr_en_mask), enable_shift_masks)
                    commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False if mask is not None else True, name=enable_shift_masks, joint_write=True))
                if digital_injection is True:  # write EnableDigInj last
                    # write DIGHITIN_SEL since after mask writing it is disabled
                    self.register.set_global_register_value("DIGHITIN_SEL", 1)
                    commands.extend(self.register.get_commands("WrRegister", name=["DIGHITIN_SEL"]))
            else:  # set masks to default values
                if disable_shift_masks:
                    map(lambda mask_name: self.register.set_pixel_register_value(mask_name, 1), disable_shift_masks)
                    commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name=disable_shift_masks, joint_write=True))
                if enable_shift_masks:
                    map(lambda mask_name: self.register.set_pixel_register_value(mask_name, 0), enable_shift_masks)
                    commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name=enable_shift_masks, joint_write=True))
                if digital_injection is True:  # write EnableDigInj last
                    # write DIGHITIN_SEL since after mask writing it is disabled
                    self.register.set_global_register_value("DIGHITIN_SEL", 1)
                    commands.extend(self.register.get_commands("WrRegister", name=["DIGHITIN_SEL"]))
            self.register_utils.send_commands(commands)
            logging.info('%d injection(s): mask step %d %s', repeat_command, mask_step, ('[%d - %d]' % (enable_mask_steps[0], enable_mask_steps[-1])) if len(enable_mask_steps) > 1 else ('[%d]' % enable_mask_steps[0]))

            if same_mask_for_all_dc:
                if fast_dc_loop:  # fast DC loop with optimized pixel register writing
                    # set repeat, should be 1 by default when arriving here
                    self.dut['TX']['CMD_REPEAT'] = repeat_command

                    # get DC command for the first DC in the list, DC command is byte padded
                    # fill CMD memory with DC command and scan loop command, inside the loop only overwrite DC command
                    dc_address_command = get_dc_address_command(enable_double_columns[0])
                    self.dut['TX']['START_SEQUENCE_LENGTH'] = len(dc_address_command)
                    self.register_utils.set_command(command=self.register_utils.concatenate_commands((dc_address_command, scan_loop_command), byte_padding=False))

                    for index, dc in enumerate(enable_double_columns):
                        if self.abort_run.is_set():
                            break
                        if index != 0:  # full command is already set before loop
                            # get DC command before wait to save some time
                            dc_address_command = get_dc_address_command(dc)
                            self.register_utils.wait_for_command()
                            if eol_function:
                                eol_function()  # do this after command has finished
                            # only set command after FPGA is ready
                            # overwrite only the DC command in CMD memory
                            self.register_utils.set_command(dc_address_command, set_length=False)  # do not set length here, because it was already set up before the loop

                        if bol_function:
                            bol_function()

                        self.dut['TX']['START']

                    # wait here before we go on because we just jumped out of the loop
                    self.register_utils.wait_for_command()
                    if eol_function:
                        eol_function()
                    self.dut['TX']['START_SEQUENCE_LENGTH'] = 0

                else:  # the slow DC loop allows writing commands inside bol and eol functions
                    for index, dc in enumerate(enable_double_columns):
                        if self.abort_run.is_set():
                            break
                        dc_address_command = get_dc_address_command(dc)
                        self.register_utils.send_command(dc_address_command)

                        if bol_function:
                            bol_function()

                        self.register_utils.send_command(scan_loop_command, repeat=repeat_command)

                        if eol_function:
                            eol_function()

            else:
                if fast_dc_loop:  # fast DC loop with optimized pixel register writing
                    dc = enable_double_columns[0]
                    ec = enable_columns(dc)
                    dcs = write_double_columns(dc)
                    commands = []
                    commands.append(conf_mode_command)
                    if disable_shift_masks:
                        curr_dis_mask = make_pixel_mask(steps=mask_steps, shift=mask_step, default=1, value=0, enable_columns=ec, mask=mask)
                        map(lambda mask_name: self.register.set_pixel_register_value(mask_name, curr_dis_mask), disable_shift_masks)
                        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, dcs=dcs, name=disable_shift_masks, joint_write=True))
                    if enable_shift_masks:
                        curr_en_mask = make_pixel_mask(steps=mask_steps, shift=mask_step, enable_columns=ec, mask=mask)
                        map(lambda mask_name: self.register.set_pixel_register_value(mask_name, curr_en_mask), enable_shift_masks)
                        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, dcs=dcs, name=enable_shift_masks, joint_write=True))
                    if digital_injection is True:
                        self.register.set_global_register_value("DIGHITIN_SEL", 1)
                        commands.extend(self.register.get_commands("WrRegister", name=["DIGHITIN_SEL"]))
                    self.register_utils.send_commands(commands)

                    dc_address_command = get_dc_address_command(dc)
                    self.dut['TX']['START_SEQUENCE_LENGTH'] = len(dc_address_command)
                    self.dut['TX']['CMD_REPEAT'] = repeat_command
                    self.register_utils.set_command(command=self.register_utils.concatenate_commands((dc_address_command, scan_loop_command), byte_padding=False))

                    for index, dc in enumerate(enable_double_columns):
                        if self.abort_run.is_set():
                            break
                        if index != 0:  # full command is already set before loop
                            ec = enable_columns(dc)
                            dcs = write_double_columns(dc)
                            dcs.extend(write_double_columns(enable_double_columns[index - 1]))
                            commands = []
                            commands.append(conf_mode_command)
                            if disable_shift_masks:
                                curr_dis_mask = make_pixel_mask(steps=mask_steps, shift=mask_step, default=1, value=0, enable_columns=ec, mask=mask)
                                map(lambda mask_name: self.register.set_pixel_register_value(mask_name, curr_dis_mask), disable_shift_masks)
                                commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, dcs=dcs, name=disable_shift_masks, joint_write=True))
                            if enable_shift_masks:
                                curr_en_mask = make_pixel_mask(steps=mask_steps, shift=mask_step, enable_columns=ec, mask=mask)
                                map(lambda mask_name: self.register.set_pixel_register_value(mask_name, curr_en_mask), enable_shift_masks)
                                commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, dcs=dcs, name=enable_shift_masks, joint_write=True))
                            if digital_injection is True:
                                self.register.set_global_register_value("DIGHITIN_SEL", 1)
                                commands.extend(self.register.get_commands("WrRegister", name=["DIGHITIN_SEL"]))
                            dc_address_command = get_dc_address_command(dc)

                            self.register_utils.wait_for_command()
                            if eol_function:
                                eol_function()  # do this after command has finished
                            self.register_utils.send_commands(commands)

                            self.dut['TX']['START_SEQUENCE_LENGTH'] = len(dc_address_command)
                            self.dut['TX']['CMD_REPEAT'] = repeat_command
                            self.register_utils.set_command(command=self.register_utils.concatenate_commands((dc_address_command, scan_loop_command), byte_padding=False))

                        if bol_function:
                            bol_function()

                        self.dut['TX']['START']

                    self.register_utils.wait_for_command()
                    if eol_function:
                        eol_function()
                    self.dut['TX']['START_SEQUENCE_LENGTH'] = 0

                else:
                    for index, dc in enumerate(enable_double_columns):
                        if self.abort_run.is_set():
                            break
                        ec = enable_columns(dc)
                        dcs = write_double_columns(dc)
                        if index != 0:
                            dcs.extend(write_double_columns(enable_double_columns[index - 1]))
                        commands = []
                        commands.append(conf_mode_command)
                        if disable_shift_masks:
                            curr_dis_mask = make_pixel_mask(steps=mask_steps, shift=mask_step, default=1, value=0, enable_columns=ec, mask=mask)
                            map(lambda mask_name: self.register.set_pixel_register_value(mask_name, curr_dis_mask), disable_shift_masks)
                            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, dcs=dcs, name=disable_shift_masks, joint_write=True))
                        if enable_shift_masks:
                            curr_en_mask = make_pixel_mask(steps=mask_steps, shift=mask_step, enable_columns=ec, mask=mask)
                            map(lambda mask_name: self.register.set_pixel_register_value(mask_name, curr_en_mask), enable_shift_masks)
                            commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, dcs=dcs, name=enable_shift_masks, joint_write=True))
                        if digital_injection is True:
                            self.register.set_global_register_value("DIGHITIN_SEL", 1)
                            commands.extend(self.register.get_commands("WrRegister", name=["DIGHITIN_SEL"]))
                        self.register_utils.send_commands(commands)

                        dc_address_command = get_dc_address_command(dc)
                        self.register_utils.send_command(dc_address_command)

                        if bol_function:
                            bol_function()

                        self.register_utils.send_command(scan_loop_command, repeat=repeat_command)

                        if eol_function:
                            eol_function()

    commands = []
    commands.extend(self.register.get_commands("ConfMode"))
    # write registers that were changed in scan_loop()
    commands.extend(self.register.get_commands("WrRegister", name=["DIGHITIN_SEL", "Colpr_Addr", "PlsrDAC"]))
    if restore_shift_masks:
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        commands.extend(self.register.get_commands("WrRegister", name=["DIGHITIN_SEL", "Colpr_Addr", "PlsrDAC"]))
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name=disable_shift_masks))
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name=enable_shift_masks))
        commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="EnableDigInj"))
#     commands.extend(self.register.get_commands("RunMode"))
    self.register_utils.send_commands(commands)

Response: Implementation of the mask shifting loop over double columns.


Instruction:
Can you generate the documentation for the following Python 3 function
def reset_service_records(self):
        '''Resetting Service Records

        This will reset Service Record counters. This will also bring back alive some FE where the output FIFO is stuck (no data is coming out in run mode).
        This should be only issued after power up and in the case of a stuck FIFO, otherwise the BCID counter starts jumping.
        '''
        logging.info('Resetting Service Records')
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        self.register.set_global_register_value('ReadErrorReq', 1)
        commands.extend(self.register.get_commands("WrRegister", name=['ReadErrorReq']))
        commands.extend(self.register.get_commands("GlobalPulse", Width=0))
        self.register.set_global_register_value('ReadErrorReq', 0)
        commands.extend(self.register.get_commands("WrRegister", name=['ReadErrorReq']))
        commands.extend(self.register.get_commands("RunMode"))
        commands.extend(self.register.get_commands("ConfMode"))
        self.send_commands(commands)

Response: Reset all the Service Records in the current thread.


Instruction:
Given the following Python 3 function, write the documentation
def reset_bunch_counter(self):
        '''Resetting Bunch Counter
        '''
        logging.info('Resetting Bunch Counter')
        commands = []
        commands.extend(self.register.get_commands("RunMode"))
        commands.extend(self.register.get_commands("BCR"))
        self.send_commands(commands)
        time.sleep(0.1)
        commands = []
        commands.extend(self.register.get_commands("ConfMode"))
        self.send_commands(commands)

Response: Reset the Bunch Counter


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate_threshold_mask(hist):
    '''Masking array elements when equal 0.0 or greater than 10 times the median

    Parameters
    ----------
    hist : array_like
        Input data.

    Returns
    -------
    masked array
        Returns copy of the array with masked elements.
    '''
    masked_array = np.ma.masked_values(hist, 0)
    masked_array = np.ma.masked_greater(masked_array, 10 * np.ma.median(hist))
    logging.info('Masking %d pixel(s)', np.ma.count_masked(masked_array))
    return np.ma.getmaskarray(masked_array)

Response: Mask array elements when equal 0. 0 or greater than 10 times the median


Instruction:
Can you generate the documentation for the following Python 3 function
def unique_row(array, use_columns=None, selected_columns_only=False):
    '''Takes a numpy array and returns the array reduced to unique rows. If columns are defined only these columns are taken to define a unique row.
    The returned array can have all columns of the original array or only the columns defined in use_columns.
    Parameters
    ----------
    array : numpy.ndarray
    use_columns : list
        Index of columns to be used to define a unique row
    selected_columns_only : bool
        If true only the columns defined in use_columns are returned

    Returns
    -------
    numpy.ndarray
    '''
    if array.dtype.names is None:  # normal array has no named dtype
        if use_columns is not None:
            a_cut = array[:, use_columns]
        else:
            a_cut = array
        if len(use_columns) > 1:
            b = np.ascontiguousarray(a_cut).view(np.dtype((np.void, a_cut.dtype.itemsize * a_cut.shape[1])))
        else:
            b = np.ascontiguousarray(a_cut)
        _, index = np.unique(b, return_index=True)
        if not selected_columns_only:
            return array[np.sort(index)]  # sort to preserve order
        else:
            return a_cut[np.sort(index)]  # sort to preserve order
    else:  # names for dtype founnd --> array is recarray
        names = list(array.dtype.names)
        if use_columns is not None:
            new_names = [names[i] for i in use_columns]
        else:
            new_names = names
        a_cut, index = np.unique(array[new_names], return_index=True)
        if not selected_columns_only:
            return array[np.sort(index)]  # sort to preserve order
        else:
            return array[np.sort(index)][new_names]

Response: Takes a numpy array and returns the array reduced to unique rows.


Instruction:
Given the following Python 3 function, write the documentation
def get_ranges_from_array(arr, append_last=True):
    '''Takes an array and calculates ranges [start, stop[. The last range end is none to keep the same length.

    Parameters
    ----------
    arr : array like
    append_last: bool
        If True, append item with a pair of last array item and None.

    Returns
    -------
    numpy.array
        The array formed by pairs of values by the given array.

    Example
    -------
    >>> a = np.array((1,2,3,4))
    >>> get_ranges_from_array(a, append_last=True)
    array([[1, 2],
           [2, 3],
           [3, 4],
           [4, None]])
    >>> get_ranges_from_array(a, append_last=False)
    array([[1, 2],
           [2, 3],
           [3, 4]])
    '''
    right = arr[1:]
    if append_last:
        left = arr[:]
        right = np.append(right, None)
    else:
        left = arr[:-1]
    return np.column_stack((left, right))

Response: Takes an array and calculates ranges [ start stop ].


Instruction:
Create a Python 3 function to
return true if ar1 and ar2 are in 1 - dimensional order.

Response:
def in1d_sorted(ar1, ar2):
    """
    Does the same than np.in1d but uses the fact that ar1 and ar2 are sorted. Is therefore much faster.

    """
    if ar1.shape[0] == 0 or ar2.shape[0] == 0:  # check for empty arrays to avoid crash
        return []
    inds = ar2.searchsorted(ar1)
    inds[inds == len(ar2)] = 0
    return ar2[inds] == ar1


Instruction:
Here you have a function in Python 3, explain what it does
def central_difference(x, y):
    '''Returns the dy/dx(x) via central difference method

    Parameters
    ----------
    x : array like
    y : array like

    Returns
    -------
    dy/dx : array like
    '''
    if (len(x) != len(y)):
        raise ValueError("x, y must have the same length")
    z1 = np.hstack((y[0], y[:-1]))
    z2 = np.hstack((y[1:], y[-1]))
    dx1 = np.hstack((0, np.diff(x)))
    dx2 = np.hstack((np.diff(x), 0))
    return (z2 - z1) / (dx2 + dx1)

Response: Returns the dy and dx of the central difference of x and y.


Instruction:
How would you code a function in Python 3 to
take 2D point data x and y and creates a profile histogram similar to the TProfile in ROOT.

Response:
def get_profile_histogram(x, y, n_bins=100):
    '''Takes 2D point data (x,y) and creates a profile histogram similar to the TProfile in ROOT. It calculates
    the y mean for every bin at the bin center and gives the y mean error as error bars.

    Parameters
    ----------
    x : array like
        data x positions
    y : array like
        data y positions
    n_bins : int
        the number of bins used to create the histogram
    '''
    if len(x) != len(y):
        raise ValueError('x and y dimensions have to be the same')
    y = y.astype(np.float32)
    n, bin_edges = np.histogram(x, bins=n_bins)  # needed to calculate the number of points per bin
    sy = np.histogram(x, bins=n_bins, weights=y)[0]  # the sum of the bin values
    sy2 = np.histogram(x, bins=n_bins, weights=y * y)[0]  # the quadratic sum of the bin values
    bin_centers = (bin_edges[1:] + bin_edges[:-1]) / 2  # calculate the bin center for all bins
    mean = sy / n  # calculate the mean of all bins
    std = np.sqrt((sy2 / n - mean * mean))  # TODO: not understood, need check if this is really the standard deviation
    std_mean = std / np.sqrt((n - 1))
    mean[np.isnan(mean)] = 0.
    std_mean[np.isnan(std_mean)] = 0.
    return bin_centers, mean, std_mean


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_rate_normalization(hit_file, parameter, reference='event', cluster_file=None, plot=False, chunk_size=500000):
    ''' Takes different hit files (hit_files), extracts the number of events or the scan time (reference) per scan parameter (parameter)
    and returns an array with a normalization factor. This normalization factor has the length of the number of different parameters.
    If a cluster_file is specified also the number of cluster per event are used to create the normalization factor.

    Parameters
    ----------
    hit_files : string
    parameter : string
    reference : string
    plot : bool

    Returns
    -------
    numpy.ndarray
    '''

    logging.info('Calculate the rate normalization')
    with tb.open_file(hit_file, mode="r+") as in_hit_file_h5:  # open the hit file
        meta_data = in_hit_file_h5.root.meta_data[:]
        scan_parameter = get_scan_parameter(meta_data)[parameter]
        event_numbers = get_meta_data_at_scan_parameter(meta_data, parameter)['event_number']  # get the event numbers in meta_data where the scan parameter changes
        event_range = get_ranges_from_array(event_numbers)
        normalization_rate = []
        normalization_multiplicity = []
        try:
            event_range[-1, 1] = in_hit_file_h5.root.Hits[-1]['event_number'] + 1
        except tb.NoSuchNodeError:
            logging.error('Cannot find hits table')
            return

        # calculate rate normalization from the event rate for triggered data / measurement time for self triggered data for each scan parameter
        if reference == 'event':
            n_events = event_range[:, 1] - event_range[:, 0]  # number of events for every parameter setting
            normalization_rate.extend(n_events)
        elif reference == 'time':
            time_start = get_meta_data_at_scan_parameter(meta_data, parameter)['timestamp_start']
            time_spend = np.diff(time_start)
            time_spend = np.append(time_spend, meta_data[-1]['timestamp_stop'] - time_start[-1])  # TODO: needs check, add last missing entry
            normalization_rate.extend(time_spend)
        else:
            raise NotImplementedError('The normalization reference ' + reference + ' is not implemented')

        if cluster_file:  # calculate the rate normalization from the mean number of hits per event per scan parameter, needed for beam data since a beam since the multiplicity is rarely constant
            cluster_table = in_hit_file_h5.root.Cluster
            index_event_number(cluster_table)
            index = 0  # index where to start the read out, 0 at the beginning, increased during looping, variable for read speed up
            best_chunk_size = chunk_size  # variable for read speed up
            total_cluster = 0
            progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=cluster_table.shape[0], term_width=80)
            progress_bar.start()
            for start_event, stop_event in event_range:  # loop over the selected events
                readout_cluster_len = 0  # variable to calculate a optimal chunk size value from the number of hits for speed up
                n_cluster_per_event = None
                for clusters, index in data_aligned_at_events(cluster_table, start_event_number=start_event, stop_event_number=stop_event, start_index=index, chunk_size=best_chunk_size):
                    if n_cluster_per_event is None:
                        n_cluster_per_event = analysis_utils.get_n_cluster_in_events(clusters['event_number'])[:, 1]  # array with the number of cluster per event, cluster per event are at least 1
                    else:
                        n_cluster_per_event = np.append(n_cluster_per_event, analysis_utils.get_n_cluster_in_events(clusters['event_number'])[:, 1])
                    readout_cluster_len += clusters.shape[0]
                    total_cluster += clusters.shape[0]
                    progress_bar.update(index)
                best_chunk_size = int(1.5 * readout_cluster_len) if int(1.05 * readout_cluster_len) < chunk_size else chunk_size  # to increase the readout speed, estimated the number of hits for one read instruction
                normalization_multiplicity.append(np.mean(n_cluster_per_event))
            progress_bar.finish()
            if total_cluster != cluster_table.shape[0]:
                logging.warning('Analysis shows inconsistent number of cluster (%d != %d). Check needed!', total_cluster, cluster_table.shape[0])

    if plot:
        x = scan_parameter
        if reference == 'event':
            plotting.plot_scatter(x, normalization_rate, title='Events per ' + parameter + ' setting', x_label=parameter, y_label='# events', log_x=True, filename=os.path.splitext(hit_file)[0] + '_n_event_normalization.pdf')
        elif reference == 'time':
            plotting.plot_scatter(x, normalization_rate, title='Measuring time per GDAC setting', x_label=parameter, y_label='time [s]', log_x=True, filename=os.path.splitext(hit_file)[0] + '_time_normalization.pdf')
        if cluster_file:
            plotting.plot_scatter(x, normalization_multiplicity, title='Mean number of particles per event', x_label=parameter, y_label='number of hits per event', log_x=True, filename=os.path.splitext(hit_file)[0] + '_n_particles_normalization.pdf')
    if cluster_file:
        normalization_rate = np.array(normalization_rate)
        normalization_multiplicity = np.array(normalization_multiplicity)
        return np.amax(normalization_rate * normalization_multiplicity).astype('f16') / (normalization_rate * normalization_multiplicity)
    return np.amax(np.array(normalization_rate)).astype('f16') / np.array(normalization_rate)

Response: Calculates the rate normalization factor for each hit in the hit file.


Instruction:
Create a Python 3 function to
take a list of files and searches for the parameter name in the file name and returns a dict with the file name Insights and the corresponding parameter value in the second dimension.

Response:
def get_parameter_value_from_file_names(files, parameters=None, unique=False, sort=True):
    """
    Takes a list of files, searches for the parameter name in the file name and returns a ordered dict with the file name
    in the first dimension and the corresponding parameter value in the second.
    The file names can be sorted by the parameter value, otherwise the order is kept. If unique is true every parameter is unique and
    mapped to the file name that occurred last in the files list.

    Parameters
    ----------
    files : list of strings
    parameter : string or list of strings
    unique : bool
    sort : bool

    Returns
    -------
    collections.OrderedDict

    """
#     unique=False
    logging.debug('Get the parameter: ' + str(parameters) + ' values from the file names of ' + str(len(files)) + ' files')
    files_dict = collections.OrderedDict()
    if parameters is None:  # special case, no parameter defined
        return files_dict
    if isinstance(parameters, basestring):
        parameters = (parameters, )
    search_string = '_'.join(parameters)
    for _ in parameters:
        search_string += r'_(-?\d+)'
    result = {}
    for one_file in files:
        parameter_values = re.findall(search_string, one_file)
        if parameter_values:
            if isinstance(parameter_values[0], tuple):
                parameter_values = list(reduce(lambda t1, t2: t1 + t2, parameter_values))
            parameter_values = [[int(i), ] for i in parameter_values]  # convert string value to list with int
            files_dict[one_file] = dict(zip(parameters, parameter_values))
            if unique:  # reduce to the files with different scan parameters
                for key, value in files_dict.items():
                    if value not in result.values():
                        result[key] = value
            else:
                result[one_file] = files_dict[one_file]
    return collections.OrderedDict(sorted(result.iteritems(), key=itemgetter(1)) if sort else files_dict)