Instruction:
Explain what the following Python 3 code does
def _parse_services(self, service_config: dict, service_name: str,
                        service_list: dict) -> dict:
        """Parse the docker compose file.

        Args:
            service_config (dict): Service configurations from the compose file
            service_name (string): Name of the services
            service_list (dict): Service configuration list

        Returns:
            dict, service specifications extracted from the compose file

        """
        for key, value in service_list['services'][service_name].items():
            service_config[key] = value
            if 'command' in key:
                key = "args"
                service_config['args'] = value
                service_config.pop('command')
            if 'ports' in key:
                endpoint_spec = self._parse_ports(value)
                service_config['endpoint_spec'] = endpoint_spec
                service_config.pop('ports')
            if 'volumes' in key:
                volume_spec = self._parse_volumes(value)
                service_config['mounts'] = volume_spec
                service_config.pop('volumes')
            if 'deploy' in key:
                self._parse_deploy(value, service_config)
                service_config.pop('deploy')
            if 'networks' in key:
                network_spec = self._parse_networks(service_list)
                service_config['networks'] = network_spec
            if 'logging' in key:
                self._parse_logging(value, service_config)
                service_config.pop('logging')
            if 'environment' in key:
                service_config['env'] = value
                service_config.pop('environment')

        # LOG.info('Service Config: %s', service_config)
        return service_config

Response: Parse the docker compose file.


Instruction:
Explain what the following Python 3 code does
def _parse_deploy(self, deploy_values: dict, service_config: dict):
        # Initialising empty dictionary
        mode = {}

        for d_value in deploy_values:
            if 'restart_policy' in d_value:
                restart_spec = docker.types.RestartPolicy(
                    **deploy_values[d_value])
                service_config['restart_policy'] = restart_spec
            if 'placement' in d_value:
                for constraints_key, constraints_value in \
                        deploy_values[d_value].items():
                    service_config[constraints_key] = constraints_value
            if 'mode' in d_value:
                mode[d_value] = deploy_values[d_value]
            if 'replicas' in d_value:
                mode[d_value] = deploy_values[d_value]
            if 'resources' in d_value:
                resource_spec = self._parse_resources(
                    deploy_values, d_value)
                service_config['resources'] = resource_spec

        # Setting the types
        mode_spec = docker.types.ServiceMode(**mode)
        service_config['mode'] = mode_spec

Response: Parse the deploy key.


Instruction:
Write a Python 3 function for
parsing the ports key. and return a Ports specification which contains exposed ports

Response:
def _parse_ports(port_values: dict) -> dict:
        """Parse ports key.

        Args:
            port_values (dict): ports configuration values

        Returns:
            dict, Ports specification which contains exposed ports

        """
        # Initialising empty dictionary
        endpoints = {}

        for port_element in port_values:
            target_port = port_element.split(':')
            for port in target_port:
                endpoints[int(port)] = int(port)

        # Setting the types
        endpoint_spec = docker.types.EndpointSpec(ports=endpoints)
        return endpoint_spec


Instruction:
Given the following Python 3 function, write the documentation
def _parse_volumes(volume_values: dict) -> str:
        for v_values in volume_values:
            for v_key, v_value in v_values.items():
                if v_key == 'source':
                    if v_value == '.':
                        source = os.path.dirname(
                            os.path.abspath(__file__))
                    else:
                        source = v_value
                if v_key == 'target':
                    target = v_value
            volume_spec = [source + ':' + target]
            return volume_spec

Response:
Parse volumes key.

        Args:
            volume_values (dict): volume configuration values

        Returns:
            string, volume specification with mount source and container path


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _parse_resources(resource_values: dict, resource_name: str) -> dict:
        # Initialising empty dictionary
        resources = {}

        for r_values in resource_values[resource_name]:
            if 'limits' in r_values:
                for r_key, r_value in \
                        resource_values[resource_name][r_values].items():
                    if 'cpu' in r_key:
                        cpu_value = float(r_value) * 10 ** 9
                        cpu_key = r_key[:3] + '_limit'
                        resources[cpu_key] = int(cpu_value)
                    if 'mem' in r_key:
                        mem_value = re.sub('M', '', r_value)
                        mem_key = r_key[:3] + '_limit'
                        resources[mem_key] = int(mem_value) * 1048576
        resources_spec = docker.types.Resources(**resources)

        return resources_spec

Response: Parse the resource key. nco.


Instruction:
Implement a function in Python 3 to
parse network key. nco.

Response:
def _parse_networks(service_list: dict) -> list:
        """Parse network key.

        Args:
            service_list (dict): Service configurations

        Returns:
            list, List of networks

        """
        # Initialising empty list
        networks = []

        for n_values in service_list['networks'].values():
            for n_key, n_value in n_values.items():
                if 'name' in n_key:
                    networks.append(n_value)
        return networks


Instruction:
Implement a Python 3 function for
parsing log key. Args : log_values - logging configuration values

Response:
def _parse_logging(log_values: dict, service_config: dict):
        """Parse log key.

        Args:
            log_values (dict): logging configuration values
            service_config (dict): Service specification
        """
        for log_key, log_value in log_values.items():
            if 'driver' in log_key:
                service_config['log_driver'] = log_value
            if 'options' in log_key:
                service_config['log_driver_options'] = log_value


Instruction:
Given the following Python 3 function, write the documentation
def _init_queue():
        LOG.info('Initialising Processing Block queue.')
        queue = ProcessingBlockQueue()
        active_pb_ids = ProcessingBlockList().active
        LOG.info('Initialising PC PB queue: %s', active_pb_ids)
        for pb_id in active_pb_ids:
            pb = ProcessingBlock(pb_id)
            queue.put(pb.id, pb.priority, pb.type)
        return queue

Response: Initialise the Processing Block queue from the database.


Instruction:
Can you tell what is the following Python 3 function doing
def _monitor_events(self):
        LOG.info("Starting to monitor PB events")
        check_counter = 0
        while True:
            if check_counter == 50:
                check_counter = 0
                LOG.debug('Checking for PB events...')

            published_events = self._pb_events.get_published_events()
            for event in published_events:
                if event.type == 'status_changed':
                    LOG.info('PB status changed event: %s',
                             event.data['status'])

                    if event.data['status'] == 'created':
                        LOG.info('Acknowledged PB created event (%s) for %s, '
                                 '[timestamp: %s]', event.id,
                                 event.object_id, event.timestamp)
                        pb = ProcessingBlock(event.object_id)
                        self._queue.put(event.object_id, pb.priority, pb.type)

                    if event.data['status'] == 'completed':
                        LOG.info('Acknowledged PB completed event (%s) for %s,'
                                 ' [timestamp: %s]', event.id,
                                 event.object_id, event.timestamp)
                        self._num_pbcs -= 1
                        if self._num_pbcs < 0:
                            self._num_pbcs = 0

            time.sleep(0.1)
            check_counter += 1

Response: Watch for Processing Block events.


Instruction:
How would you explain what the following Python 3 function does
def _processing_controller_status(self):
        LOG.info('Starting Processing Block queue reporter.')
        while True:
            LOG.info('PB queue length = %d', len(self._queue))
            time.sleep(self._report_interval)
            if active_count() != 5:
                LOG.critical('Processing Controller not running '
                             'correctly! (%d/%d threads active)',
                             active_count(), 5)

Response: Report on the status of the Processing Block queue.


Instruction:
Explain what the following Python 3 code does
def _schedule_processing_blocks(self):
        LOG.info('Starting to Schedule Processing Blocks.')
        while True:
            time.sleep(0.5)
            if not self._queue:
                continue
            if self._num_pbcs >= self._max_pbcs:
                LOG.warning('Resource limit reached!')
                continue
            _inspect = Inspect(app=APP)
            if self._queue and _inspect.active() is not None:
                next_pb = self._queue[-1]
                LOG.info('Considering %s for execution...', next_pb[2])
                utc_now = datetime.datetime.utcnow()
                time_in_queue = (utc_now -
                                 datetime_from_isoformat(next_pb[4]))
                if time_in_queue.total_seconds() >= 10:
                    item = self._queue.get()
                    LOG.info('------------------------------------')
                    LOG.info('>>> Executing %s! <<<', item)
                    LOG.info('------------------------------------')
                    execute_processing_block.delay(item)
                    self._num_pbcs += 1
                else:
                    LOG.info('Waiting for resources for %s', next_pb[2])

Response: Schedule processing blocks for execution.


Instruction:
Here you have a function in Python 3, explain what it does
def _monitor_pbc_status(self):
        LOG.info('Starting to Monitor PBC status.')
        inspect = celery.current_app.control.inspect()
        workers = inspect.ping()
        start_time = time.time()
        while workers is None:
            time.sleep(0.1)
            elapsed = time.time() - start_time
            if elapsed > 20.0:
                LOG.warning('PBC not found!')
                break
        if workers is not None:
            for worker in workers:
                _tasks = inspect.registered_tasks()[worker]
                LOG.info('Worker: %s tasks:', worker)
                for task_index, task_name in enumerate(_tasks):
                    LOG.info('  %02d : %s', task_index, task_name)

        while True:
            LOG.info('Checking PBC status (%d/%d)', self._num_pbcs,
                     self._max_pbcs)
            celery_app = celery.current_app
            inspect = celery_app.control.inspect()
            workers = inspect.ping()
            if workers is None:
                LOG.warning('PBC service not found!')
            else:
                LOG.info('PBC state:  %s', celery_app.events.State())
                _active = inspect.active()
                _scheduled = inspect.scheduled()
                for worker in workers:
                    LOG.info('  Worker %s: scheduled: %s, active: %s',
                             worker, _active[worker], _scheduled[worker])
            time.sleep(self._report_interval)

Response: Monitor the PBC status.


Instruction:
Given the following Python 3 function, write the documentation
def start(self):
        # TODO(BMo) having this check is probably a good idea but I've \
        # disabled it for now while the PBC is in flux.
        # assert sip_pbc.release.__version__ == '1.2.3'

        scheduler_threads = [
            Thread(target=self._monitor_events, daemon=True),
            Thread(target=self._processing_controller_status, daemon=True),
            Thread(target=self._schedule_processing_blocks, daemon=True),
            Thread(target=self._monitor_pbc_status, daemon=True)
        ]

        for thread in scheduler_threads:
            thread.start()

        try:
            for thread in scheduler_threads:
                thread.join()
        except KeyboardInterrupt:
            LOG.info('Keyboard interrupt!')
            sys.exit(0)
        finally:
            LOG.info('Finally!')

Response: Start the scheduler threads.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def alarm():
    if request.method == 'POST':
        response = {'message': 'POST Accepted'}
        logging.info('alarm POSTED!')
        data = request.data
        logging.info(data)
        string = json.dumps(data)
        producer.send('SIP-alarms', string.encode())
        return response
    return ""

Response: A alarm is a POST request.


Instruction:
Write a Python 3 function for
updating the current state of a service.

Response:
def _update_service_current_state(service: ServiceState):
    """Update the current state of a service.

    Updates the current state of services after their target state has changed.

    Args:
        service (ServiceState): Service state object to update

    """
    LOG.debug("Setting current state from target state for %s", service.id)
    service.update_current_state(service.target_state)


Instruction:
Make a summary of the following Python 3 code
def _update_services_instant_gratification(sdp_target_state: str):
    service_states = get_service_state_list()

    # Set the target state of services
    for service in service_states:
        if service.current_state != sdp_target_state:
            LOG.debug('Setting the current state of %s to be %s', service.id,
                      sdp_target_state)
            service.update_current_state(sdp_target_state)

Response: For demonstration purposes only.


Instruction:
Can you write a function in Python 3 where it
updates the target states of services based on the SDP target state.

Response:
def _update_services_target_state(sdp_target_state: str):
    """Update the target states of services based on SDP target state.

    When we get a new target state this function is called to ensure
    components receive the target state(s) and/or act on them.

    Args:
        sdp_target_state (str): Target state of SDP

    """
    service_states = get_service_state_list()

    # Set the target state of services
    for service in service_states:
        if service.current_state != sdp_target_state:
            LOG.debug('Setting the target state of %s to be %s', service.id,
                      sdp_target_state)
            service.update_target_state(sdp_target_state)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _handle_sdp_target_state_updated(sdp_state: SDPState):
    LOG.info('Handling SDP target state updated event...')
    LOG.info('SDP target state: %s', sdp_state.target_state)

    # Map between the SDP target state and the service target state?
    if sdp_state.target_state == 'off':
        _update_services_target_state('off')

    # TODO: Work out if the state of SDP has reached the target state.

    # If yes, update the current state.
    sdp_state.update_current_state(sdp_state.target_state)

Response: Handle an SDP target state change event.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _init(sdp_state: SDPState):
    # Parse command line arguments.
    LOG.info("Initialising: %s", __service_id__)

    # FIXME(BMo) There is a bug when SDP or services 'start' in the 'off'
    # state. At the moment it is impossible to transition out of this.

    # FIXME(BMo) **Hack** Register all services or if already registered do
    # nothing (this is handled by the ServiceState object).
    _services = [
        "ExecutionControl:AlarmReceiver:1.0.0",
        "ExecutionControl:AlertManager:1.0.0",
        "ExecutionControl:ConfigurationDatabase:5.0.1",
        "ExecutionControl:MasterController:1.3.0",
        "ExecutionControl:ProcessingController:1.2.6",
        "ExecutionControl:ProcessingBlockController:1.3.0",
        "TangoControl:Database:1.0.4",
        "TangoControl:MySQL:1.0.3",
        "TangoControl:SDPMaster:1.2.1",
        "TangoControl:Subarrays:1.2.0",
        "TangoControl:ProcessingBlocks:1.2.0",
        "Platform:Kafka:2.1.1",
        "Platform:Prometheus:1.0.0",
        "Platform:PrometheusPushGateway:0.7.0",
        "Platform:RedisCommander:210.0.0",
        "Platform:Zookeeper:3.4.13"
    ]
    for service_id in _services:
        subsystem, name, version = service_id.split(':')
        ServiceState(subsystem, name, version)

    # If the SDP state is 'unknown', mark the SDP state as init.
    # FIXME(BMo) This is not right as we want to allow for recovery from
    # failure without just reinitialising...!? ie. respect the old sate
    # NOTE: If the state is 'off' we will want to reset the database
    # with 'skasip_config_db_init --clear'
    if sdp_state.current_state in ['unknown', 'off']:
        try:
            LOG.info("Setting the SDPState to 'init'")
            sdp_state.update_current_state('init', force=True)
        except ValueError as error:
            LOG.critical('Unable to set the State of SDP to init! %s',
                         str(error))

    LOG.info("Updating Service States")
    service_state_list = get_service_state_list()

    # FIXME(BMo) **Hack** Mark all Services in the 'unknown' state as
    # initialising.
    for service_state in service_state_list:
        if service_state.current_state in ['unknown', 'off']:
            service_state.update_current_state('init', force=True)

    # FIXME(BMo) **Hack** After 'checking' that the services are 'on' set
    # their state on 'on' after a short delay.
    # FIXME(BMo) This check should not be serialised!!! (should be part of the
    # event loop)
    for service_state in service_state_list:
        if service_state.current_state == 'init':
            time.sleep(random.uniform(0, 0.2))
            service_state.update_current_state('on')

    # FIXME(BMo): **Hack** Now the all services are on, set the sate of SDP to
    # 'standby'
    # FIXME(BMo) This should also be part of the event loop.
    services_on = [service.current_state == 'on'
                   for service in service_state_list]
    if all(services_on):
        LOG.info('All Services are online!.')
        sdp_state.update_current_state('standby')
    else:
        LOG.critical('Master Controller failed to initialise.')

    return service_state_list

Response: Initialize the master controller service.


Instruction:
Create a Python 3 function to
process an event from the SDP state machine.

Response:
def _process_event(event: Event, sdp_state: SDPState,
                   service_states: List[ServiceState]):
    """Process a SDP state change event."""
    LOG.debug('Event detected! (id : "%s", type: "%s", data: "%s")',
              event.object_id, event.type, event.data)

    if event.object_id == 'SDP' and event.type == 'current_state_updated':
        LOG.info('SDP current state updated, no action required!')

    if event.object_id == 'SDP' and event.type == 'target_state_updated':
        LOG.info("SDP target state changed to '%s'",
                 sdp_state.target_state)

        # If the sdp is already in the target state do nothing
        if sdp_state.target_state == sdp_state.current_state:
            LOG.warning('SDP already in %s state',
                        sdp_state.current_state)
            return

        # Check that a transition to the target state is allowed in the
        # current state.
        if not sdp_state.is_target_state_allowed(sdp_state.target_state):
            LOG.error('Transition to %s is not allowed when in state %s',
                      sdp_state.target_state, sdp_state.current_state)
            sdp_state.target_state = sdp_state.current_state
            return

        _update_services_target_state(sdp_state.target_state)

        # If asking SDP to turn off, also turn off services.
        if sdp_state.target_state == 'off':
            LOG.info('Turning off services!')
            for service_state in service_states:
                service_state.update_target_state('off')
                service_state.update_current_state('off')

        LOG.info('Processing target state change request ...')
        time.sleep(0.1)
        LOG.info('Done processing target state change request!')

        # Assuming that the SDP has responding to the target
        # target state command by now, set the current state
        # to the target state.
        sdp_state.update_current_state(sdp_state.target_state)

        if sdp_state.current_state == 'alarm':
            LOG.debug('raising SDP state alarm')
            SIP_STATE_ALARM.set(1)
        else:
            SIP_STATE_ALARM.set(0)
        try:
            # FIXME(BMo) the pushgateway host should not be hardcoded!
            push_to_gateway('platform_pushgateway:9091', job='SIP',
                            registry=COLLECTOR_REGISTRY)
        except urllib.error.URLError:
            LOG.warning("Unable to connect to the Alarms service!")


Instruction:
Explain what the following Python 3 code does
def _process_state_change_events():
    sdp_state = SDPState()
    service_states = get_service_state_list()
    state_events = sdp_state.get_event_queue(subscriber=__service_name__)
    state_is_off = sdp_state.current_state == 'off'

    counter = 0
    while True:
        time.sleep(0.1)
        if not state_is_off:

            # *Hack* to avoid problems with historical events not being
            # correctly handled by EventQueue.get(), replay old events every
            # 10s
            # - see issue #54
            if counter % 1000 == 0:
                LOG.debug('Checking published events ... %d', counter / 1000)
                _published_events = state_events.get_published_events(
                    process=True)
                for _state_event in _published_events:
                    _process_event(_state_event, sdp_state, service_states)
            else:
                _state_event = state_events.get()
                if _state_event:
                    _process_event(_state_event, sdp_state, service_states)
                    state_is_off = sdp_state.current_state == 'off'
            counter += 1

Response: This function processes events relating to the overall state of SDP and responds to SDP state change events.


Instruction:
Can you generate the documentation for the following Python 3 function
def main():
    # Parse command line args.
    _parse_args()
    LOG.info("Starting: %s", __service_id__)

    # Subscribe to state change events.
    # FIXME(BMo) This API is unfortunate as it looks like we are only
    # subscribing to sdp_state events.
    LOG.info('Subscribing to state change events (subscriber = %s)',
             __service_name__)
    sdp_state = SDPState()
    _ = sdp_state.subscribe(subscriber=__service_name__)

    # Initialise the service.
    _ = _init(sdp_state)
    LOG.info('Finished initialising!')

    # Enter a pseudo event-loop (using Sched) to monitor for state change
    # events
    # (Also random set services into a fault or alarm state if enabled)
    LOG.info('Responding to state change events ...')
    try:
        _process_state_change_events()
    except ValueError as error:
        LOG.critical('Value error: %s', str(error))
    except KeyboardInterrupt as err:
        LOG.debug('Keyboard Interrupt %s', err)
    LOG.info('Exiting!')

Response: Merge temp_main and main.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def main():
    stream_config = spead2.send.StreamConfig(
        max_packet_size=16356, rate=1000e6, burst_size=10, max_heaps=1)
    item_group = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48, 0))

    # Add item descriptors to the heap.
    num_baselines = (512 * 513) // 2
    dtype = [('TCI', 'i1'), ('FD', 'u1'), ('VIS', '<c8', 4)]
    item_group.add_item(
        id=0x6000, name='visibility_timestamp_count', description='',
        shape=tuple(), format=None, dtype='<u4')
    item_group.add_item(
        id=0x6001, name='visibility_timestamp_fraction', description='',
        shape=tuple(), format=None, dtype='<u4')
    item_group.add_item(
        id=0x6005, name='visibility_baseline_count', description='',
        shape=tuple(), format=None, dtype='<u4')
    item_group.add_item(
        id=0x6008, name='scan_id', description='',
        shape=tuple(), format=None, dtype='<u8')
    item_group.add_item(
        id=0x600A, name='correlator_output_data', description='',
        shape=(num_baselines,), dtype=dtype)

    # Create streams and send start-of-stream message.
    streams = []
    num_streams = 2
    for i in range(num_streams):
        stream = spead2.send.UdpStream(
            thread_pool=spead2.ThreadPool(threads=1),
            hostname='127.0.0.1', port=41000 + i, config=stream_config)
        stream.send_heap(item_group.get_start())
        streams.append(stream)

    vis = numpy.zeros(shape=(num_baselines,), dtype=dtype)
    num_heaps = 200
    start_time = time.time()
    for stream in streams:
        # Update values in the heap.
        item_group['visibility_timestamp_count'].value = 1
        item_group['visibility_timestamp_fraction'].value = 0
        item_group['visibility_baseline_count'].value = num_baselines
        item_group['scan_id'].value = 100000000
        item_group['correlator_output_data'].value = vis
        # Iterate heaps.
        for i in range(num_heaps):
            # Send heap.
            stream.send_heap(item_group.get_heap(descriptors='all', data='all'))

    # Print time taken.
    duration = time.time() - start_time
    data_size = num_streams * num_heaps * (vis.nbytes / 1e6)
    print("Sent %.3f MB in %.3f sec (%.3f MB/sec)" % (
        data_size, duration, (data_size/duration)))

    # Send end-of-stream message.
    for stream in streams:
        stream.send_heap(item_group.get_end())

Response: Runs the test sender.


Instruction:
Can you create a Python 3 function that
creates an event object from an event dictionary object.

Response:
def from_config(cls, config: dict):
        """Create an event object from an event dictionary object.

        Args:
            config (dict): Event Configuration dictionary.

        """
        timestamp = config.get('timestamp', None)
        return cls(config.get('id'),
                   config.get('type'),
                   config.get('data', dict()),
                   config.get('origin', None),
                   timestamp,
                   config.get('object_type', None),
                   config.get('object_id', None),
                   config.get('object_key', None))


Instruction:
Can you create a Python 3 function that
reads visibility data from a Measurement Set and updates the grid normalisation.

Response:
def process_input_data(filename, imager, grid_data, grid_norm, grid_weights):
    """Reads visibility data from a Measurement Set.

    The visibility grid or weights grid is updated accordingly.

    Visibility data are read from disk in blocks of size num_baselines.

    Args:
        filename (str):                    Name of Measurement Set to open.
        imager (oskar.Imager):             Handle to configured imager.
        grid_data (numpy.ndarray or None): Visibility grid to populate.
        grid_norm (float)                  Current grid normalisation.
        grid_weights (numpy.ndarray):      Weights grid to populate or read.

    Returns:
        grid_norm (float):                 Updated grid normalisation.
    """
    # Get data from the input Measurement Set.
    ms = oskar.MeasurementSet.open(filename)
    block_start = 0
    num_rows = ms.num_rows
    num_baselines = ms.num_stations * (ms.num_stations - 1) // 2

    # Loop over data blocks of size num_baselines.
    while block_start < num_rows:
        block_size = num_rows - block_start
        if block_size > num_baselines:
            block_size = num_baselines

        # Get the baseline coordinates. (Replace this with a query to LTS.)
        uvw = ms.read_column('UVW', block_start, block_size)

        # Read the Stokes-I visibility weights.
        vis_weights = ms.read_column('WEIGHT', block_start, block_size)
        if ms.num_pols == 4:
            vis_weights = 0.5 * (vis_weights[:, 0] + vis_weights[:, 3])

        # Loop over frequency channels.
        # (We expect there to be only one channel here, but loop just in case.)
        for j in range(ms.num_channels):
            # Get coordinates in wavelengths.
            coords = uvw * (ms.freq_start_hz + j * ms.freq_inc_hz) / 299792458.

            # Get the Stokes-I visibilities for this channel.
            vis_data = None
            if not imager.coords_only:
                vis_data = ms.read_vis(block_start, j, 1, block_size)
                if ms.num_pols == 4:
                    vis_data = 0.5 * (vis_data[0, :, 0] + vis_data[0, :, 3])

            # Update the grid plane with this visibility block.
            grid_norm = imager.update_plane(
                coords[:, 0], coords[:, 1], coords[:, 2], vis_data,
                vis_weights, grid_data, grid_norm, grid_weights)

        # Increment start row by block size.
        block_start += block_size

    # Return updated grid normalisation.
    return grid_norm


Instruction:
Given the following Python 3 function, write the documentation
def main():
    # Check command line arguments.
    if len(sys.argv) < 2:
        raise RuntimeError(
            'Usage: mpiexec -n <np> '
            'python mpi_imager_test.py <settings_file> <dir>')

    # Get the MPI communicator and initialise broadcast variables.
    comm = MPI.COMM_WORLD
    settings = None
    inputs = None
    grid_weights = None

    # Create log.
    log = logging.getLogger()
    log.setLevel(logging.DEBUG)
    if len(log.handlers) == 0:
        log.addHandler(logging.StreamHandler(sys.stdout))

    if comm.Get_rank() == 0:
        # Load pipeline settings.
        with open(sys.argv[1]) as f:
            settings = json.load(f)

        # Get a list of input Measurement Sets to process.
        data_dir = str(sys.argv[2])
        inputs = glob(join(data_dir, '*.ms')) + glob(join(data_dir, '*.MS'))
        inputs = filter(None, inputs)
        log.info('Found input Measurement Sets: %s', ', '.join(inputs))

        # Distribute the list of Measurement Sets among processors.
        inputs = chunks(inputs, comm.Get_size())

    # Broadcast settings and scatter list of input files.
    comm.barrier()
    settings = comm.bcast(settings)
    inputs = comm.scatter(inputs)

    # Record which file(s) this node is working on.
    log.debug('Rank %d, processing [%s]', comm.Get_rank(), ', '.join(inputs))

    # Create an imager and configure it.
    precision = settings['precision']
    imager = oskar.Imager(precision)
    for key, value in settings['imager'].items():
        setattr(imager, key, value)

    # Allocate a local visibility grid.
    grid_norm = 0.
    grid_dim = [imager.plane_size, imager.plane_size]
    grid_data = numpy.zeros(grid_dim,
                            dtype='c8' if precision == 'single' else 'c16')

    # Process data according to mode.
    if settings['combine']:
        if imager.weighting == 'Uniform' or imager.algorithm == 'W-projection':
            # If necessary, generate a local weights grid.
            local_weights = None
            if imager.weighting == 'Uniform':
                grid_weights = numpy.zeros(grid_dim, dtype=precision)
                local_weights = numpy.zeros(grid_dim, dtype=precision)

            # Do a first pass for uniform weighting or W-projection.
            imager.coords_only = True
            for f in inputs:
                log.info('Reading coordinates from %s', f)
                process_input_data(f, imager, None, 0.0, local_weights)
            imager.coords_only = False

            # Get maximum number of W-projection planes required.
            num_w_planes = imager.num_w_planes
            num_w_planes = comm.allreduce(num_w_planes, op=MPI.MAX)
            imager.num_w_planes = num_w_planes

            # Combine (reduce) weights grids, and broadcast the result.
            if local_weights is not None:
                comm.Allreduce(local_weights, grid_weights, op=MPI.SUM)

        # Populate the local visibility grid.
        for f in inputs:
            log.info('Reading visibilities from %s', f)
            grid_norm = process_input_data(f, imager, grid_data, grid_norm,
                                           grid_weights)

        # Combine (reduce) visibility grids.
        grid = numpy.zeros_like(grid_data)
        comm.Reduce(grid_data, grid, op=MPI.SUM)
        grid_norm = comm.reduce(grid_norm, op=MPI.SUM)

        # Finalise grid and save image.
        if comm.Get_rank() == 0:
            save_image(imager, grid, grid_norm, settings['output_file'])
            log.info('Finished. Output file is %s', settings['output_file'])
    else:
        for f in inputs:
            # Clear the grid.
            grid_norm = 0.
            grid_data.fill(0)
            if imager.weighting == 'Uniform':
                grid_weights = numpy.zeros(grid_dim, dtype=precision)

            # Do a first pass for uniform weighting or W-projection.
            if imager.weighting == 'Uniform' or \
                    imager.algorithm == 'W-projection':
                imager.coords_only = True
                log.info('Reading coordinates from %s', f)
                process_input_data(f, imager, None, 0.0, grid_weights)
                imager.coords_only = False

            # Populate the local visibility grid.
            log.info('Reading visibilities from %s', f)
            grid_norm = process_input_data(f, imager, grid_data, grid_norm,
                                           grid_weights)

            # Save image by finalising grid.
            output_file = splitext(f)[0] + '.fits'
            save_image(imager, grid_data, grid_norm, output_file)
            log.info('Finished. Output file is %s', output_file)

Response: Runs test imaging pipeline using MPI.


Instruction:
Create a Python 3 function to
get list of Scheduling Blocks Instances known to SDP.

Response:
def get():
    """Return list of Scheduling Blocks Instances known to SDP ."""
    LOG.debug('GET list of SBIs.')

    # Construct response object.
    _url = get_root_url()
    response = dict(scheduling_blocks=[],
                    links=dict(home='{}'.format(_url)))

    # Get ordered list of SBI ID's.
    block_ids = DB.get_sched_block_instance_ids()

    # Loop over SBIs and add summary of each to the list of SBIs in the
    # response.
    for block in DB.get_block_details(block_ids):
        block_id = block['id']
        LOG.debug('Adding SBI %s to list', block_id)
        LOG.debug(block)

        block['num_processing_blocks'] = len(block['processing_block_ids'])

        temp = ['OK'] * 10 + ['WAITING'] * 4 + ['FAILED'] * 2
        block['status'] = choice(temp)
        try:
            del block['processing_block_ids']
        except KeyError:
            pass
        block['links'] = {
            'detail': '{}/scheduling-block/{}' .format(_url, block_id)
        }
        response['scheduling_blocks'].append(block)
    return response, HTTPStatus.OK


Instruction:
Create a Python 3 function for
providing table of scheduling block instance metadata for use with AJAX tables

Response:
def get_table():
    """Provides table of scheduling block instance metadata for use with AJAX
    tables"""
    response = dict(blocks=[])
    block_ids = DB.get_sched_block_instance_ids()
    for index, block_id in enumerate(block_ids):
        block = DB.get_block_details([block_id]).__next__()
        info = [
            index,
            block['id'],
            block['sub_array_id'],
            len(block['processing_blocks'])
        ]
        response['blocks'].append(info)
    return response, HTTPStatus.OK


Instruction:
How would you explain what the following Python 3 function does
def set_value(self, key, field, value):
        self._db.hset(key, field, value)

Response: Add the state of the key and field to the state of the key and field


Instruction:
Can you create a Python 3 function that
pushes the specified event to the tail of the list stored at the key event_name with the specified event_type and block_id.

Response:
def push_event(self, event_name, event_type, block_id):
        """Push inserts all the specified values at the tail of the list
        stored at the key"""
        self._db.rpush(event_name, dict(type=event_type, id=block_id))


Instruction:
Can you tell what is the following Python 3 function doing
def save_dict(self, key: str, my_dict: dict, hierarchical: bool = False):
        for _key, _value in my_dict.items():
            if isinstance(_value, dict):
                if not hierarchical:
                    self._db.hmset(key, {_key: json.dumps(_value)})
                else:
                    self.save_dict(key + ':' + _key, _value, hierarchical)
            elif isinstance(_value, list):
                if not hierarchical:
                    self._db.hmset(key, {_key: str(_value)})
                else:
                    print('saving list at ', key + ':' + _key)
                    self._db.lpush(key + ':' + _key, *_value[::-1])
            elif isinstance(_value, bool):
                self._db.hmset(key, {_key: str(_value)})
            else:
                self._db.hmset(key, {_key: _value})

Response: Store the specified dictionary at the specified key.


Instruction:
Make a summary of the following Python 3 code
def _build_dict(my_dict, keys, values):
        temp = my_dict
        for depth, key in enumerate(keys):
            if depth < len(keys) - 1:
                if key not in temp:
                    temp[key] = dict()
                temp = temp[key]
            else:
                if key not in temp:
                    temp[key] = values
                else:
                    temp[key] = {**temp[key], **values}
        return my_dict

Response: Build a dictionary from a set of redis hashes.


Instruction:
Can you tell what is the following Python 3 function doing
def _load_values(self, db_key: str) -> dict:
        if self._db.type(db_key) == 'list':
            db_values = self._db.lrange(db_key, 0, -1)
            for i, value in enumerate(db_values):
                try:
                    db_values[i] = ast.literal_eval(value)
                except SyntaxError:
                    pass
                except ValueError:
                    pass
        else:  # self._db.type == 'hash'
            db_values = self._db.hgetall(db_key)
            for _key, _value in db_values.items():
                try:
                    db_values[_key] = ast.literal_eval(_value)
                except SyntaxError:
                    pass
                except ValueError:
                    pass
        return db_values

Response: Load values from the db at the specified key.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _load_dict_hierarchical(self, db_key: str) -> dict:
        db_keys = self._db.keys(pattern=db_key + '*')
        my_dict = {}
        for _db_key in db_keys:
            if self._db.type(_db_key) == 'list':
                db_values = self._db.lrange(_db_key, 0, -1)
                for i, value in enumerate(db_values):
                    try:
                        db_values[i] = ast.literal_eval(value)
                    except SyntaxError:
                        pass
                    except ValueError:
                        pass
            else:  # self._db.type == 'hash'
                db_values = self._db.hgetall(_db_key)
                for _key, _value in db_values.items():
                    try:
                        db_values[_key] = ast.literal_eval(_value)
                    except SyntaxError:
                        pass
                    except ValueError:
                        pass
            my_dict = self._build_dict(my_dict, _db_key.split(':'),
                                       db_values)
        return my_dict[db_key]

Response: Load a dictionary stored hierarchically at db_key.


Instruction:
Here you have a function in Python 3, explain what it does
def load_dict(self, db_key: str, hierarchical: bool = False) -> dict:
        if not hierarchical:
            db_values = self._db.hgetall(db_key)
            for _key, _value in db_values.items():
                if isinstance(_value, str):
                    db_values[_key] = ast.literal_eval(_value)
            my_dict = db_values
        else:
            my_dict = self._load_dict_hierarchical(db_key)
        return my_dict

Response: Load the dictionary at the specified key.


Instruction:
Create a Python 3 function for
loading values from a dictionary with the specified dict_keys.

Response:
def load_dict_values(self, db_key: str, dict_keys: List[str],
                         hierarchical: bool = False) -> List:
        """Load values from a dictionary with the specified dict_keys.

        Args:
            db_key (str): Key where the dictionary is stored
            dict_keys (List[str]): Keys within the dictionary to load.
            hierarchical (bool): If True, expect the dictionary to have been
                stored hierarchically. If False, expect the dictionary to have
                been stored flat.

        Returns:
            object: The value stored at dict_key in the dictionary stored at
            key

        """
        result = []
        if not hierarchical:
            _values = self._db.hmget(db_key, *dict_keys)
            result = [ast.literal_eval(_value) for _value in _values]
        else:
            # Get all keys in the set of keys for this dict 'db_key'
            db_keys = self._db.keys(pattern=db_key + '*')
            for _db_key in db_keys:
                # Check if one of the dict_keys is an entire sub-dict entry
                for name in _db_key.split(':')[1:]:
                    if name in dict_keys:
                        _values = self._load_values(_db_key)
                        result.append(_values)

                # Look in the sub-dict for any of the dict_keys
                _values = self._db.hmget(_db_key, *dict_keys)
                for i, value in enumerate(_values):
                    try:
                        _values[i] = ast.literal_eval(value)
                    except SyntaxError:
                        pass
                    except ValueError:
                        pass
                result += [value for value in _values if value is not None]
        return result


Instruction:
Write a Python 3 script to
set the value of field in a hash stored at key.

Response:
def set_hash_value(self, key, field, value, pipeline=False):
        """Set the value of field in a hash stored at key.

        Args:
            key (str): key (name) of the hash
            field (str): Field within the hash to set
            value: Value to set
            pipeline (bool): True, start a transaction block. Default false.

        """
        # FIXME(BMo): new name for this function -> save_dict_value ?
        if pipeline:
            self._pipeline.hset(key, field, str(value))
        else:
            self._db.hset(key, field, str(value))


Instruction:
Can you generate the documentation for the following Python 3 function
def prepend_to_list(self, key, *value, pipeline=False):
        if pipeline:
            self._pipeline.lpush(key, *value)
        else:
            self._db.lpush(key, *value)

Response: Prepends a new element to the beginning of the list stored at key.


Instruction:
Write a Python 3 script for
adding a new element to the end of the list stored at key.

Response:
def append_to_list(self, key, *value, pipeline=False):
        """Add new element to the end of the list stored at key.

        Args:
            key (str): Key where the list is stored
            value: Value to add to the list
            pipeline (bool): True, start a transaction block. Default false.

        """
        if pipeline:
            self._pipeline.rpush(key, *value)
        else:
            self._db.rpush(key, *value)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_list(self, key, pipeline=False):
        if pipeline:
            return self._pipeline.lrange(key, 0, -1)

        return self._db.lrange(key, 0, -1)

Response: Get all the values in the list stored at key.


Instruction:
Explain what the following Python 3 code does
def delete(self, *names: str, pipeline=False):
        if pipeline:
            self._pipeline.delete(*names)
        else:
            self._db.delete(*names)

Response: Delete one or more keys specified by names.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_event(self, event_name, event_history=None):
        if event_history is None:
            event_history = event_name + '_history'
        return self._db.rpoplpush(event_name, event_history)

Response: Get an event from the database.


Instruction:
Can you implement a function in Python 3 that
removes specified value from the list stored at key.

Response:
def remove_from_list(self, key: str, value, count: int = 0,
                         pipeline: bool = False):
        """Remove specified value(s) from the list stored at key.

        Args:
            key (str): Key where the list is stored.
            value: value to remove
            count (int): Number of entries to remove, default 0 == all
            pipeline(bool): If True, start a transaction block. Default False.

        """
        if pipeline:
            if redis.__version__ == '2.10.6':
                self._pipeline.lrem(name=key, value=value, num=count)
            else:
                self._pipeline.lrem(key, count, value)
        else:
            if self._db.exists(key):
                if redis.__version__ == '2.10.6':
                    self._db.lrem(name=key, value=value, num=count)
                else:
                    self._db.lrem(key, count, value)


Instruction:
Explain what the following Python 3 code does
def watch(self, key, pipeline=False):
        if pipeline:
            self._pipeline.watch(key)
        else:
            self._db.watch(key)

Response: Watch the given key for conditional execution of a transaction.


Instruction:
Implement a function in Python 3 to
post a message to a given channel.

Response:
def publish(self, channel, message, pipeline=False):
        """Post a message to a given channel.

        Args:
            channel (str): Channel where the message will be published
            message (str): Message to publish
            pipeline (bool): True, start a transaction block. Default false.

        """
        if pipeline:
            self._pipeline.publish(channel, message)
        else:
            self._db.publish(channel, message)


Instruction:
Implement a function in Python 3 to
get the health of all services.

Response:
def get_services_health(self) -> dict:
        """Get the health of all services.

        Returns:
            dict, services id and health status

        """
        # Initialise
        services_health = {}

        # Get Service IDs
        services_ids = self._get_services()

        for service_id in services_ids:
            service_name = DC.get_service_name(service_id)

            # Check if the current and actual replica levels are the same
            if DC.get_replicas(service_id) != \
                    DC.get_actual_replica(service_id):
                services_health[service_name] = "Unhealthy"
            else:
                services_health[service_name] = "Healthy"

        return services_health


Instruction:
Explain what the following Python 3 code does
def get_overall_services_health(self) -> str:
        services_health_status = self.get_services_health()

        # Evaluate overall health
        health_status = all(status == "Healthy" for status in
                            services_health_status.values())

        # Converting from bool to str
        if health_status:
            overall_status = "Healthy"
        else:
            overall_status = "Unhealthy"

        return overall_status

Response: Get the overall health of all the services.


Instruction:
Write a Python 3 script to
get the health of a service using service_id.

Response:
def get_service_health(service_id: str) -> str:
        """Get the health of a service using service_id.

        Args:
            service_id

        Returns:
            str, health status

        """
        # Check if the current and actual replica levels are the same
        if DC.get_replicas(service_id) != DC.get_actual_replica(service_id):
            health_status = "Unhealthy"
        else:
            health_status = "Healthy"

        return health_status


Instruction:
Explain what the following Python 3 code does
def get():
    uptime = time.time() - START_TIME
    response = dict(uptime=f'{uptime:.2f}s',
                    links=dict(root='{}'.format(get_root_url())))

    # TODO(BM) check if we can connect to the config database ...
    # try:
    #     DB.get_sub_array_ids()
    # except ConnectionError as error:
    #     response['state'] = 'ERROR'
    #     response['message'] = str(error)
    return response, HTTPStatus.OK

Response: Check the health of this service


Instruction:
Can you write a function in Python 3 where it
processes the received heaps and store the data in the internal buffer.

Response:
def process_buffer(self, i_block, receive_buffer):
        """Blocking function to process the received heaps.
        This is run in an executor.
        """
        self._log.info("Worker thread processing block %i", i_block)
        time_overall0 = time.time()
        time_unpack = 0.0
        time_write = 0.0
        for i_heap, heap in enumerate(receive_buffer.result()):
            # Skip and log any incomplete heaps.
            if isinstance(heap, spead2.recv.IncompleteHeap):
                self._log.info("Dropped incomplete heap %i", heap.cnt + 1)
                continue

            # Update the item group from this heap.
            items = self._item_group.update(heap)

            # Get the time and channel indices from the heap index.
            i_chan = i_heap // self._num_buffer_times
            i_time = i_heap % self._num_buffer_times

            if 'correlator_output_data' in items:
                vis_data = items['correlator_output_data'].value['VIS']
                if self._block is None:
                    num_baselines = vis_data.shape[0]
                    num_pols = vis_data[0].shape[0]
                    self._block = numpy.zeros((self._num_buffer_times,
                                               self._num_streams,
                                               num_baselines),
                                              dtype=('c8', num_pols))
                    self._block[:, :, :] = 0  # To make the copies faster.

                # Unpack data from the heap into the block to be processed.
                time_unpack0 = time.time()
                self._block[i_time, i_chan, :] = vis_data
                time_unpack += time.time() - time_unpack0

                # Check the data for debugging!
                val = self._block[i_time, i_chan, -1][-1].real
                self._log.debug("Data: %.3f", val)

        if self._block is not None:
            # Process the buffered data here.
            if self._config['process_data']:
                pass

            # Write the buffered data to storage.
            if self._config['write_data']:
                time_write0 = time.time()
                with open(self._config['filename'], 'ab') as f:
                    # Don't use pickle, it's really slow (even protocol 4)!
                    numpy.save(f, self._block, allow_pickle=False)
                time_write += time.time() - time_write0

        # Report time taken.
        time_overall = time.time() - time_overall0
        self._log.info("Total processing time: %.1f ms", 1000 * time_overall)
        self._log.info("Unpack was %.1f %%", 100 * time_unpack / time_overall)
        self._log.info("Write was %.1f %%", 100 * time_write / time_overall)
        if time_unpack != 0.0:
            self._log.info("Memory speed %.1f MB/s",
                           (self._block.nbytes * 1e-6) / time_unpack)
        if time_write != 0.0:
            self._log.info("Write speed %.1f MB/s",
                           (self._block.nbytes * 1e-6) / time_write)


Instruction:
Can you generate the documentation for the following Python 3 function
async def _run_loop(self, executor):
        loop = asyncio.get_event_loop()

        # Get first heap in each stream (should be empty).
        self._log.info("Waiting for %d streams to start...", self._num_streams)
        for stream in self._streams:
            await stream.get(loop=loop)

        i_block = 0
        receive_buffer = [None, None]
        while True:
            # Process the previous buffer, if available.
            processing_tasks = None
            if i_block > 0:
                i_buffer_proc = (i_block - 1) % 2
                processing_tasks = loop.run_in_executor(
                    executor, self.process_buffer, i_block - 1,
                    receive_buffer[i_buffer_proc])

            # Set up asynchronous receive on all streams.
            i_buffer_recv = i_block % 2
            receive_tasks = []
            for stream in self._streams:
                for _ in range(self._num_buffer_times):
                    # NOTE: loop.create_task() is needed to schedule these
                    # in the right order!
                    receive_tasks.append(
                        loop.create_task(stream.get(loop=loop)))
            receive_buffer[i_buffer_recv] = asyncio.gather(*receive_tasks)

            # Ensure asynchronous receives and previous processing tasks are
            # done.
            self._log.info("Receiving block %i", i_block)
            try:
                await receive_buffer[i_buffer_recv]
                self._log.info("Received block %i", i_block)
            except spead2.Stopped:
                self._log.info("Stream Stopped")
                break
            if processing_tasks:
                await processing_tasks

            i_block += 1

Response: Main loop for the main loop.


Instruction:
Implement a Python 3 function for
closing the buffer and return the buffer.

Response:
def close(self):
        """Add docstring!"""

        # Seek to the start of the buffer
        self.seek(0)
        while True:

            # Copy bytes from the buffer until we reach the end of the JSON
            brace_count = 0
            quoted = False
            json_string = ''
            while True:

                # Read a character
                b = self.read(1)
                if b == b'':
                    return
                c = b.decode()

                # If it is a \ then copy the next character
                if c == '\\':
                    json_string += c
                    json_string += self.read(1).decode()
                    continue

                # If we are inside quotes we just need to detect a
                # closing quote
                if c == '"':
                    if quoted:
                        quoted = False
                    else:
                        quoted = True

                # Otherwise we count the braces
                if c == '{':
                    brace_count += 1
                elif c == '}':
                    brace_count -= 1

                # Copy the character into the JSON string
                json_string += c

                # If the brace count is zero we are done
                if brace_count == 0:
                    break

            # Parse the JSON so that we can get the size of the data array
            meta = json.loads(json_string)
            n_channels = meta['data_cube']['n_channels']
            n_sub_integrations = meta['data_cube']['n_sub_integrations']

            # Save the JSON to a file
            f = open('{0}_{1}_{2}.json'.format(
                meta['metadata']['observation_id'],
                meta['metadata']['beam_id'],
                meta['metadata']['name']), 'w')

            f.write(json.dumps(meta))
            f.close()

            # Read the data
            data = self.read(n_channels * n_sub_integrations)

            # Write it to a file
            f = open('{0}_{1}_{2}.data'.format(
                meta['metadata']['observation_id'],
                meta['metadata']['beam_id'],
                meta['metadata']['name']), 'wb')
            f.write(data)
            f.close()


Instruction:
How would you explain what the following Python 3 function does
def run(self):
        self._log.info('Starting Pulsar Search Interface')
        # Instantiate a dummy authorizer for managing 'virtual' users
        authorizer = DummyAuthorizer()

        # Define a new user having full r/w permissions and a read-only
        # anonymous user
        authorizer.add_user(self._config['login']['user'],
                            self._config['login']['psswd'], '.',
                            perm=self._config['login']['perm'])
        authorizer.add_anonymous(os.getcwd())

        # Instantiate FTP handler class
        handler = FTPHandler
        handler.authorizer = authorizer
        handler.abstracted_fs = PulsarFileSystem

        # Define a customized banner (string returned when client connects)
        handler.banner = "SKA SDP pulsar search interface."

        # Instantiate FTP server class and listen on 0.0.0.0:7878
        address = (self._config['address']['listen'],
                   self._config['address']['port'])
        server = FTPServer(address, handler)

        # set a limit for connections
        server.max_cons = 256
        server.max_cons_per_ip = 5

        # start ftp server
        server.serve_forever()

Response: Start the FTP server for pulsar search.


Instruction:
Can you write a function in Python 3 where it
returns the workflow stage status.

Response:
def status(self) -> str:
        """Return the workflow stage status."""
        # As status is a modifiable property, have to reload from the db.
        self._config = self._load_config()
        return self._config.get('status')


Instruction:
Implement a Python 3 function for
setting the workflow stage status.

Response:
def status(self, value):
        """Set the workflow stage status."""
        # FIXME(BM) This is currently a hack because workflow stages
        #           don't each have their own db entry.
        pb_key = SchedulingObject.get_key(PB_KEY, self._pb_id)
        stages = DB.get_hash_value(pb_key, 'workflow_stages')
        stages = ast.literal_eval(stages)
        stages[self._index]['status'] = value
        DB.set_hash_value(pb_key, 'workflow_stages', stages)


Instruction:
Make a summary of the following Python 3 code
def _load_config(self):
        pb_key = SchedulingObject.get_key(PB_KEY, self._pb_id)
        stages = DB.get_hash_value(pb_key, 'workflow_stages')
        stages = ast.literal_eval(stages)
        return stages[self._index]

Response: Load the workflow stage config from the database.


Instruction:
Make a summary of the following Python 3 code
def register_pb_devices(num_pbs: int = 100):
    tango_db = Database()
    LOG.info("Registering PB devices:")
    dev_info = DbDevInfo()
    # pylint: disable=protected-access
    dev_info._class = 'ProcessingBlockDevice'
    dev_info.server = 'processing_block_ds/1'

    for index in range(num_pbs):
        dev_info.name = 'sip_sdp/pb/{:05d}'.format(index)
        LOG.info("\t%s", dev_info.name)
        tango_db.add_device(dev_info)

Response:
Register PBs devices.

    Note(BMo): Ideally we do not want to register any devices here. There
    does not seem to be a way to create a device server with no registered
    devices in Tango. This is (probably) because Tango devices must have been
    registered before the server starts ...


Instruction:
Create a Python 3 function for
creating an SBI object from the specified configuration dictionary.

Response:
def from_config(cls, config_dict: dict, schema_path: str = None):
        """Create an SBI object from the specified configuration dict.

        NOTE(BM) This should really be done as a single atomic db transaction.

        Args:
            config_dict(dict): SBI configuration dictionary
            schema_path(str, optional): Path to the SBI config schema.

        """
        # Validate the SBI config schema
        if schema_path is None:
            schema_path = join(dirname(__file__), 'schema',
                               'configure_sbi.json')
        with open(schema_path, 'r') as file:
            schema = json.loads(file.read())
            validate(config_dict, schema)

        # Add SBI status field
        config_dict['status'] = 'created'

        # Set the subarray field to None if not defined.
        if 'subarray_id' not in config_dict:
            config_dict['subarray_id'] = 'None'

        # Add created, and updated timestamps.
        timestamp = datetime.datetime.utcnow().isoformat()
        config_dict['created'] = timestamp
        config_dict['updated'] = timestamp

        # Split out the processing block data array
        pb_list = copy.deepcopy(config_dict['processing_blocks'])

        # Remove processing blocks from the SBI configuration.
        config_dict.pop('processing_blocks', None)

        # Add list of PB ids to the SBI configuration
        config_dict['processing_block_ids'] = []
        for pb in pb_list:
            config_dict['processing_block_ids'].append(pb['id'])

        # Add the SBI data object to the database.
        key = SchedulingObject.get_key(SBI_KEY, config_dict['id'])
        DB.save_dict(key, config_dict, hierarchical=False)
        # DB.set_hash_values(key, config_dict)

        # Add the SBI id to the list of active SBIs
        key = '{}:active'.format(SBI_KEY)
        DB.append_to_list(key, config_dict['id'])

        # Publish notification to subscribers
        sbi = SchedulingObject(SBI_KEY, config_dict['id'])
        sbi.set_status('created')

        for pb in pb_list:
            pb['sbi_id'] = config_dict['id']
            cls._add_pb(pb)

        return cls(config_dict['id'])


Instruction:
Write a Python 3 function for
aborting the SBI and associated PBs.

Response:
def abort(self):
        """Abort the SBI (and associated PBs)."""
        self.set_status('aborted')
        DB.remove_from_list('{}:active'.format(self._type), self._id)
        DB.append_to_list('{}:aborted'.format(self._type), self._id)
        sbi_pb_ids = ast.literal_eval(
            DB.get_hash_value(self._key, 'processing_block_ids'))

        for pb_id in sbi_pb_ids:
            pb = ProcessingBlock(pb_id)
            pb.abort()


Instruction:
Implement a function in Python 3 to
return the list of PB ids associated with the SBI.

Response:
def get_pb_ids(self) -> List[str]:
        """Return the list of PB ids associated with the SBI.

        Returns:
            list, Processing block ids

        """
        values = DB.get_hash_value(self._key, 'processing_block_ids')
        return ast.literal_eval(values)


Instruction:
Here you have a function in Python 3, explain what it does
def get_id(date=None, project: str = 'sip',
               instance_id: int = None) -> str:
        """Get a SBI Identifier.

        Args:
            date (str or datetime.datetime, optional): UTC date of the SBI
            project (str, optional ): Project Name
            instance_id (int, optional): SBI instance identifier

        Returns:
            str, Scheduling Block Instance (SBI) ID.

        """
        if date is None:
            date = datetime.datetime.utcnow()

        if isinstance(date, datetime.datetime):
            date = date.strftime('%Y%m%d')

        if instance_id is None:
            instance_id = randint(0, 9999)

        return 'SBI-{}-{}-{:04d}'.format(date, project, instance_id)

Response: Get a SBI Identifier.


Instruction:
Here you have a function in Python 3, explain what it does
def _add_pb(pb_config: dict):
        # Add status field to the PB
        pb_config['status'] = 'created'

        # Add created and updated timestamps to the PB
        timestamp = datetime.datetime.utcnow().isoformat()
        pb_config['created'] = timestamp
        pb_config['updated'] = timestamp

        # set default priority, if not defined
        if 'priority' not in pb_config:
            pb_config['priority'] = 0

        # Retrieve the workflow definition
        SchedulingBlockInstance._update_workflow_definition(pb_config)

        # If needed, add resources and dependencies fields
        keys = ['resources_required', 'resources_assigned', 'dependencies']
        for key in keys:
            if key not in pb_config:
                pb_config[key] = []
            for stage in pb_config['workflow_stages']:
                if key not in stage:
                    stage[key] = []

        # Add PB to the the database
        key = SchedulingObject.get_key(PB_KEY, pb_config['id'])
        # DB.set_hash_values(key, pb_config)
        DB.save_dict(key, pb_config, hierarchical=False)

        # Add to list of PB ids
        key = '{}:active'.format(PB_KEY)
        DB.append_to_list(key, pb_config['id'])
        key = '{}:active:{}'.format(PB_KEY, pb_config['type'])
        DB.append_to_list(key, pb_config['id'])

        # Publish an event to to notify subscribers of the new PB
        pb = SchedulingObject(PB_KEY, pb_config['id'])
        pb.set_status('created')

Response: Add a PB to the database.


Instruction:
Can you create a Python 3 function that
updates the PB configuration workflow definition.

Response:
def _update_workflow_definition(pb_config: dict):
        """Update the PB configuration workflow definition.

        Args:
            pb_config (dict): PB configuration dictionary

        Raises:
            RunTimeError, if the workflow definition (id, version)
            specified in the sbi_config is not known.

        """
        known_workflows = get_workflows()
        workflow_id = pb_config['workflow']['id']
        workflow_version = pb_config['workflow']['version']
        if workflow_id not in known_workflows or \
           workflow_version not in known_workflows[workflow_id]:
            raise RuntimeError("Unknown workflow definition: {}:{}"
                               .format(workflow_id, workflow_version))
        workflow = get_workflow(workflow_id, workflow_version)
        for stage in workflow['stages']:
            stage['status'] = 'none'
        pb_config['workflow_parameters'] = pb_config['workflow']['parameters']
        pb_config['workflow_id'] = pb_config['workflow']['id']
        pb_config['workflow_version'] = pb_config['workflow']['version']
        pb_config['workflow_stages'] = workflow['stages']
        pb_config.pop('workflow', None)


Instruction:
Here you have a function in Python 3, explain what it does
def root():
    response = {
        "links": {
            "message": "Welcome to the SIP Processing Controller Interface",
            "items": [
                {"href": "{}health".format(request.url)},
                {"href": "{}subarrays".format(request.url)},
                {"href": "{}scheduling_blocks".format(request.url)},
                {"href": "{}processing_blocks".format(request.url)}
            ]
        }
    }
    return response, HTTPStatus.OK

Response: Placeholder root url for the PCI.


Instruction:
Can you write a function in Python 3 where it
generates a scheduling_block id

Response:
def generate_scheduling_block_id(num_blocks, project='test'):
    """Generate a scheduling_block id"""
    _date = strftime("%Y%m%d", gmtime())
    _project = project
    for i in range(num_blocks):
        yield '{}-{}-sbi{:03d}'.format(_date, _project, i)


Instruction:
Can you implement a function in Python 3 that
adds a scheduling block to the database returning a response object

Response:
def add_scheduling_block(config):
    """Adds a scheduling block to the database, returning a response object"""
    try:
        DB.add_sbi(config)
    except jsonschema.ValidationError as error:
        error_dict = error.__dict__
        for key in error_dict:
            error_dict[key] = error_dict[key].__str__()
        error_response = dict(message="Failed to add scheduling block",
                              reason="JSON validation error",
                              details=error_dict)
        return error_response, HTTPStatus.BAD_REQUEST
    response = dict(config=config,
                    message='Successfully registered scheduling block '
                            'instance with ID: {}'.format(config['id']))
    response['links'] = {
        'self': '{}scheduling-block/{}'.format(request.url_root,
                                               config['id']),
        'list': '{}'.format(request.url),
        'home': '{}'.format(request.url_root)
    }
    return response, HTTPStatus.ACCEPTED


Instruction:
Can you generate a brief explanation for the following Python 3 code
def missing_db_response(func):
    @wraps(func)
    def with_exception_handling(*args, **kwargs):
        """Wrapper to check for connection failures"""
        try:
            return func(*args, **kwargs)
        except ConnectionError as error:
            return (dict(error='Unable to connect to Configuration Db.',
                         error_message=str(error),
                         links=dict(root='{}'.format(get_root_url()))),
                    HTTPStatus.NOT_FOUND)
    return with_exception_handling

Response: Decorator to wrap a function to check for connection failures and return a response dict.


Instruction:
How would you code a function in Python 3 to
run the workflow task.

Response:
def main():
    """Run the workflow task."""
    log = logging.getLogger('sip.mock_workflow_stage')

    if len(sys.argv) != 2:
        log.critical('Expecting JSON string as first argument!')
        return

    config = json.loads(sys.argv[1])
    log.info('Running mock_workflow_stage (version: %s).', __version__)
    log.info('Received configuration: %s', json.dumps(config))
    log.info('Starting task')

    i = 0
    start_time = time.time()
    duration = config.get('duration', 20)
    while time.time() - start_time <= duration:
        time.sleep(duration / 20)
        elapsed = time.time() - start_time
        log.info("  %s %2i / 20 (elapsed %.2f s)",
                 config.get('message', 'Progress '),
                 i + 1, elapsed)
        i += 1

    log.info('Task complete!')


Instruction:
Can you generate a brief explanation for the following Python 3 code
def main(sleep_length=0.1):
    log = logging.getLogger('sip.examples.log_spammer')

    log.info('Starting to spam log messages every %fs', sleep_length)
    counter = 0
    try:
        while True:
            log.info('Hello %06i (log_spammer: %s, sip logging: %s)',
                     counter, _version.__version__, __version__)
            counter += 1
            time.sleep(sleep_length)
    except KeyboardInterrupt:
        log.info('Exiting...')

Response: Main function for spam log messages


Instruction:
Make a summary of the following Python 3 code
def main():
    init_logging()

    LOG.info("Starting imaging-pipeline")

    # Read parameters
    PARFILE = 'parameters.json'
    if len(sys.argv) > 1:
       PARFILE = sys.argv[1]
    LOG.info("JSON parameter file = %s", PARFILE)
    try:
       with open(PARFILE, "r") as par_file:
             jspar = json.load(par_file)
    except AssertionError as error:
       LOG.critical('ERROR %s', error)
       return

    # We will use dask
    arlexecute.set_client(get_dask_Client())
    arlexecute.run(init_logging)

    # Import visibility list from HDF5 file
    vis_list = import_blockvisibility_from_hdf5(
        '%s/%s' % (RESULTS_DIR, jspar["files"]["vis_list"]))

    # Now read the BlockVisibilities constructed using a model drawn from GLEAM
    predicted_vislist = import_blockvisibility_from_hdf5(
        '%s/%s' % (RESULTS_DIR, jspar["files"]["predicted_vis_list"]))
    corrupted_vislist = import_blockvisibility_from_hdf5(
        '%s/%s' % (RESULTS_DIR, jspar["files"]["corrupted_vis_list"]))

# Reproduce parameters from the visibility data
    ntimes = vis_list[0].nvis

    phasecentre = vis_list[0].phasecentre
    print(phasecentre)
    polframe = vis_list[0].polarisation_frame.type
    LOG.info("Polarisation Frame of vis_list: %s", polframe)

    wprojection_planes = jspar["advice"]["wprojection_planes"]
    guard_band_image   = jspar["advice"]["guard_band_image"]
    delA               = jspar["advice"]["delA"]
    advice_low = advise_wide_field(vis_list[0], guard_band_image=guard_band_image,
                                   delA=delA,
                                   wprojection_planes=wprojection_planes)
    advice_high = advise_wide_field(vis_list[-1], guard_band_image=guard_band_image,
                                    delA=delA,
                                    wprojection_planes=wprojection_planes)

    vis_slices = advice_low['vis_slices']
    npixel = advice_high['npixels2']
    cellsize = min(advice_low['cellsize'], advice_high['cellsize'])

# Recovering frequencies
    fstart = vis_list[0].frequency
    fend = vis_list[-1].frequency
    num_freq_win = len(vis_list)
    frequency = numpy.linspace(fstart, fend, num_freq_win)

# Recovering bandwidths
    channel_bandwidth = numpy.array(num_freq_win * [vis_list[1].frequency - vis_list[0].frequency])

    # Get the LSM. This is currently blank.
    model_list = [
        arlexecute.execute(create_image_from_visibility)(
            vis_list[f],
            npixel=npixel,
            frequency=[frequency[f]],
            channel_bandwidth=[channel_bandwidth[f]],
            cellsize=cellsize,
            phasecentre=phasecentre,
            polarisation_frame=PolarisationFrame(polframe))
        for f, freq in enumerate(frequency)
    ]
    # future_predicted_vislist = arlexecute.scatter(predicted_vislist)

    # Create and execute graphs to make the dirty image and PSF
    # LOG.info('About to run invert to get dirty image')
    # dirty_list = invert_component(future_predicted_vislist, model_list,
    #                               context='wstack',
    #                               vis_slices=vis_slices, dopsf=False)
    # dirty_list = arlexecute.compute(dirty_list, sync=True)

    # LOG.info('About to run invert to get PSF')
    # psf_list = invert_component(future_predicted_vislist, model_list,
    #                             context='wstack',
    #                             vis_slices=vis_slices, dopsf=True)
    # psf_list = arlexecute.compute(psf_list, sync=True)

    # Now deconvolve using msclean
    # LOG.info('About to run deconvolve')
    # deconvolve_list, _ = deconvolve_component(
    #     dirty_list, psf_list,
    #     model_imagelist=model_list,
    #     deconvolve_facets=8,
    #     deconvolve_overlap=16,
    #     deconvolve_taper='tukey',
    #     scales=[0, 3, 10],
    #     algorithm='msclean',
    #     niter=1000,
    #     fractional_threshold=0.1,
    #     threshold=0.1,
    #     gain=0.1,
    #     psf_support=64)
    # deconvolved = arlexecute.compute(deconvolve_list, sync=True)

    LOG.info('About to run continuum imaging')
    continuum_imaging_list = continuum_imaging_arlexecute(
        predicted_vislist,
        model_imagelist		= model_list,
        context			= jspar["processing"]["continuum_imaging"]["context"] , #'wstack',
        vis_slices		= vis_slices,
        scales			= jspar["processing"]["continuum_imaging"]["scales"],             #[0, 3, 10],
        algorithm		= jspar["processing"]["continuum_imaging"]["algorithm"],          #'mmclean',
        nmoment			= jspar["processing"]["continuum_imaging"]["nmoment"],            #3,
        niter			= jspar["processing"]["continuum_imaging"]["niter"],		  #1000,
        fractional_threshold	= jspar["processing"]["continuum_imaging"]["fractional_threshold"], #0.1,
        threshold		= jspar["processing"]["continuum_imaging"]["threshold"], 	  #0.1,
        nmajor			= jspar["processing"]["continuum_imaging"]["nmajor"], 		  #5,
        gain			= jspar["processing"]["continuum_imaging"]["gain"],               #0.25,
        deconvolve_facets	= jspar["processing"]["continuum_imaging"]["deconvolve_facets"],  #8,
        deconvolve_overlap	= jspar["processing"]["continuum_imaging"]["deconvolve_overlap"], #16,
        deconvolve_taper	= jspar["processing"]["continuum_imaging"]["deconvolve_taper"],   #'tukey',
        psf_support		= jspar["processing"]["continuum_imaging"]["psf_support"] )        #64)
    result = arlexecute.compute(continuum_imaging_list, sync=True)
    deconvolved = result[0][0]
    residual = result[1][0]
    restored = result[2][0]

    print(qa_image(deconvolved, context='Clean image - no selfcal'))
    print(qa_image(restored, context='Restored clean image - no selfcal'))
    export_image_to_fits(restored,
                         '%s/%s' % (RESULTS_DIR, jspar["files"]["continuum_imaging_restored"]))

    print(qa_image(residual[0], context='Residual clean image - no selfcal'))
    export_image_to_fits(residual[0],
                         '%s/%s' % (RESULTS_DIR, jspar["files"]["continuum_imaging_residual"]))

    controls = create_calibration_controls()

    controls['T']['first_selfcal'] = jspar["processing"]["controls"]["T"]["first_selfcal"]
    controls['G']['first_selfcal'] = jspar["processing"]["controls"]["G"]["first_selfcal"]
    controls['B']['first_selfcal'] = jspar["processing"]["controls"]["B"]["first_selfcal"]

    controls['T']['timescale'] = jspar["processing"]["controls"]["T"]["timescale"]
    controls['G']['timescale'] = jspar["processing"]["controls"]["G"]["timescale"]
    controls['B']['timescale'] = jspar["processing"]["controls"]["B"]["timescale"]

    PP.pprint(controls)

    future_corrupted_vislist = arlexecute.scatter(corrupted_vislist)
    ical_list = ical_arlexecute(future_corrupted_vislist,
        model_imagelist		= model_list,
        context			= jspar["processing"]["ical"]["context"] , 		#'wstack',
        calibration_context	= jspar["processing"]["ical"]["calibration_context"] , 	#'TG',
        controls		= controls,
        vis_slices		= ntimes,
        scales			= jspar["processing"]["ical"]["scales"],             	#[0, 3, 10],
        timeslice		= jspar["processing"]["ical"]["timeslice"],          	#'auto',
        algorithm		= jspar["processing"]["ical"]["algorithm"],          	#'mmclean',
        nmoment			= jspar["processing"]["ical"]["nmoment"],            	#3,
        niter			= jspar["processing"]["ical"]["niter"],		  	#1000,
        fractional_threshold	= jspar["processing"]["ical"]["fractional_threshold"], 	#0.1,
        threshold		= jspar["processing"]["ical"]["threshold"], 	  	#0.1,
        nmajor			= jspar["processing"]["ical"]["nmajor"], 		#5,
        gain			= jspar["processing"]["ical"]["gain"],               	#0.25,
        deconvolve_facets	= jspar["processing"]["ical"]["deconvolve_facets"],  	#8,
        deconvolve_overlap	= jspar["processing"]["ical"]["deconvolve_overlap"], 	#16,
        deconvolve_taper	= jspar["processing"]["ical"]["deconvolve_taper"],   	#'tukey',
        global_solution		= jspar["processing"]["ical"]["global_solution"],    	#False,
        do_selfcal		= jspar["processing"]["ical"]["do_selfcal"],         	#True,
        psf_support		= jspar["processing"]["ical"]["psf_support"])         	#64

    LOG.info('About to run ical')
    result = arlexecute.compute(ical_list, sync=True)
    deconvolved = result[0][0]
    residual = result[1][0]
    restored = result[2][0]

    print(qa_image(deconvolved, context='Clean image'))
    print(qa_image(restored, context='Restored clean image'))
    export_image_to_fits(restored, '%s/%s' % (RESULTS_DIR, jspar["files"]["ical_restored"]))

    print(qa_image(residual[0], context='Residual clean image'))
    export_image_to_fits(residual[0], '%s/%s' % (RESULTS_DIR, jspar["files"]["ical_residual"]))

    arlexecute.close()

Response: Main function for the imaging - pipeline.


Instruction:
Write a Python 3 script for
subbing array detail resource. This method will list scheduling blocks and processing blocks in the specified sub-array.

Response:
def get(sub_array_id):
    """Sub array detail resource.

    This method will list scheduling blocks and processing blocks
    in the specified sub-array.
    """
    if not re.match(r'^subarray-0[0-9]|subarray-1[0-5]$', sub_array_id):
        response = dict(error='Invalid sub-array ID specified "{}" does not '
                              'match sub-array ID naming convention '
                              '(ie. subarray-[00-15]).'.
                        format(sub_array_id))
        return response, HTTPStatus.BAD_REQUEST
    if sub_array_id not in DB.get_sub_array_ids():
        response = dict(error='Sub-array "{}" does not currently exist. '
                              'Known sub-arrays = {}'
                        .format(sub_array_id, DB.get_sub_array_ids()))
        return response, HTTPStatus.NOT_FOUND

    block_ids = DB.get_sub_array_sbi_ids(sub_array_id)
    _blocks = [b for b in DB.get_block_details(block_ids)]
    response = dict(scheduling_blocks=[])
    _url = get_root_url()
    for block in _blocks:
        block['links'] = {
            'self': '{}/scheduling-block/{}'.format(_url, block['id'])
        }
        response['scheduling_blocks'].append(block)
    response['links'] = {
        'self': '{}'.format(request.url),
        'list': '{}/sub-arrays'.format(_url),
        'home': '{}'.format(_url),
    }
    return response, HTTPStatus.OK


Instruction:
Can you tell what is the following Python 3 function doing
def create(sub_array_id):
    config = request.data
    config['sub_array_id'] = 'subarray-{:02d}'.format(sub_array_id)
    return add_scheduling_block(config)

Response: Create a SDP Scheduling Block instance with SDP.


Instruction:
Write a Python 3 function that can
return the list of scheduling blocks instances associated with the sub_array_id

Response:
def get_scheduling_block(sub_array_id, block_id):
    """Return the list of scheduling blocks instances associated with the sub
    array"""
    block_ids = DB.get_sub_array_sbi_ids(sub_array_id)
    if block_id in block_ids:
        block = DB.get_block_details([block_id]).__next__()
        return block, HTTPStatus.OK

    return dict(error="unknown id"), HTTPStatus.NOT_FOUND


Instruction:
Create a Python 3 function for
getting list of data files to add to the package.

Response:
def package_files(directory):
    """Get list of data files to add to the package."""
    paths = []
    for (path, _, file_names) in walk(directory):
        for filename in file_names:
            paths.append(join('..', path, filename))
    return paths


Instruction:
Make a summary of the following Python 3 code
def register_master():
    tango_db = Database()
    device = "sip_sdp/elt/master"
    device_info = DbDevInfo()
    device_info._class = "SDPMasterDevice"
    device_info.server = "sdp_master_ds/1"
    device_info.name = device
    devices = tango_db.get_device_name(device_info.server, device_info._class)
    if device not in devices:
        LOG.info('Registering device "%s" with device server "%s"',
                 device_info.name, device_info.server)
        tango_db.add_device(device_info)

Response: Register the SDP Master device.


Instruction:
How would you explain what the following Python 3 function does
def main(args=None, **kwargs):
    LOG.info('Starting %s', __service_id__)
    return run([SDPMasterDevice], verbose=True, msg_stream=sys.stdout,
               args=args, **kwargs)

Response: Run the Tango SDP Master device server.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_command_line():
    parser = argparse.ArgumentParser(
        prog='csp_pulsar_sender',
        description='Send fake pulsar data using ftp protocol.')
    parser.add_argument('config_file', type=argparse.FileType('r'),
                        help='JSON configuration file.')
    parser.add_argument('-v', '--verbose', help='Enable verbose messages.',
                        action='store_true')
    parser.add_argument('-p', '--print_settings', help='Print settings file.',
                        action='store_true')
    return parser.parse_args()

Response: Parse command line arguments.


Instruction:
Can you tell what is the following Python 3 function doing
def _init_log(level=logging.DEBUG):
    log = logging.getLogger(__file__)
    log.setLevel(level)
    handler = logging.StreamHandler(sys.stdout)
    handler.setLevel(level)
    formatter = logging.Formatter('%(asctime)s: %(message)s',
                                  '%Y/%m/%d-%H:%M:%S')
    handler.setFormatter(formatter)
    log.addHandler(handler)
    return log

Response: Initialise the logging object.


Instruction:
How would you explain what the following Python 3 function does
def wrap_subscribe(transport_layer, channel, callback, *args, **kwargs):

    return _wrap_subscription(
        transport_layer, transport_layer.subscribe, channel, callback, *args, **kwargs
    )

Response: Wrap a subscribe call in the transport layer.


Instruction:
How would you explain what the following Python 3 function does
def wrap_subscribe_broadcast(transport_layer, channel, callback, *args, **kwargs):

    return _wrap_subscription(
        transport_layer,
        transport_layer.subscribe_broadcast,
        channel,
        callback,
        *args,
        **kwargs
    )

Response:
Wrap subscribe_broadcast in a new ID
   .


Instruction:
Write a Python 3 script for
starting the transport layer.

Response:
def start_transport(self):
        """If a transport object has been defined then connect it now."""
        if self.transport:
            if self.transport.connect():
                self.log.debug("Service successfully connected to transport layer")
            else:
                raise RuntimeError("Service could not connect to transport layer")
            # direct all transport callbacks into the main queue
            self._transport_interceptor_counter = itertools.count()
            self.transport.subscription_callback_set_intercept(
                self._transport_interceptor
            )
        else:
            self.log.debug("No transport layer defined for service. Skipping.")


Instruction:
How would you code a function in Python 3 to
take a callback function and returns a function that takes headers and messages and places them on the main service queue.

Response:
def _transport_interceptor(self, callback):
        """Takes a callback function and returns a function that takes headers and
        messages and places them on the main service queue."""

        def add_item_to_queue(header, message):
            queue_item = (
                Priority.TRANSPORT,
                next(
                    self._transport_interceptor_counter
                ),  # insertion sequence to keep messages in order
                (callback, header, message),
            )
            self.__queue.put(
                queue_item
            )  # Block incoming transport until insertion completes

        return add_item_to_queue


Instruction:
Create a Python 3 function for
injecting pipes connecting the service to the frontend. Two arguments are supported.

Response:
def connect(self, frontend=None, commands=None):
        """Inject pipes connecting the service to the frontend. Two arguments are
        supported: frontend= for messages from the service to the frontend,
        and commands= for messages from the frontend to the service.
        The injection should happen before the service is started, otherwise the
        underlying file descriptor references may not be handled correctly."""
        if frontend:
            self.__pipe_frontend = frontend
            self.__send_service_status_to_frontend()
        if commands:
            self.__pipe_commands = commands


Instruction:
How would you explain what the following Python 3 function does
def extend_log(self, field, value):
        self.__log_extensions.append((field, value))
        try:
            yield
        except Exception as e:
            setattr(e, "workflows_log_" + field, value)
            raise
        finally:
            self.__log_extensions.remove((field, value))

Response:
A context wherein a specified extra field in log messages is populated
        with a fixed value. This affects all log messages within the context.


Instruction:
Create a Python 3 function to
function to continuously retrieve data from the frontend. Commands are sent to the central priority queue. If the pipe from the frontend is closed the service shutdown is initiated. Check every second if service has shut down, then terminate. This function is run by a separate daemon thread, which is started by the __start_command_queue_listener function.

Response:
def __command_queue_listener(self):
        """Function to continuously retrieve data from the frontend. Commands are
        sent to the central priority queue. If the pipe from the frontend is
        closed the service shutdown is initiated. Check every second if service
        has shut down, then terminate.
        This function is run by a separate daemon thread, which is started by
        the __start_command_queue_listener function.
        """
        self.log.debug("Queue listener thread started")
        counter = itertools.count()  # insertion sequence to keep messages in order
        while not self.__shutdown:
            if self.__pipe_commands.poll(1):
                try:
                    message = self.__pipe_commands.recv()
                except EOFError:
                    # Pipe was closed by frontend. Shut down service.
                    self.__shutdown = True
                    self.log.error(
                        "Pipe closed by frontend, shutting down service", exc_info=True
                    )
                    break
                queue_item = (Priority.COMMAND, next(counter), message)
                try:
                    self.__queue.put(queue_item, True, 60)
                except queue.Full:
                    # If the message can't be stored within 60 seconds then the service is
                    # operating outside normal parameters. Try to shut it down.
                    self.__shutdown = True
                    self.log.error(
                        "Write to service priority queue failed, shutting down service",
                        exc_info=True,
                    )
                    break
        self.log.debug("Queue listener thread terminating")


Instruction:
Can you create a Python 3 function that
starts the function __command_queue_listener in a separate thread. This function is continuously listens to the frontend connected to the frontend.

Response:
def __start_command_queue_listener(self):
        """Start the function __command_queue_listener in a separate thread. This
        function continuously listens to the pipe connected to the frontend.
        """
        thread_function = self.__command_queue_listener

        class QueueListenerThread(threading.Thread):
            def run(qltself):
                thread_function()

        assert not hasattr(self, "__queue_listener_thread")
        self.log.debug("Starting queue listener thread")
        self.__queue_listener_thread = QueueListenerThread()
        self.__queue_listener_thread.daemon = True
        self.__queue_listener_thread.name = "Command Queue Listener"
        self.__queue_listener_thread.start()


Instruction:
Explain what the following Python 3 code does
def _log_send(self, logrecord):
        for field, value in self.__log_extensions:
            setattr(logrecord, field, value)
        self.__send_to_frontend({"band": "log", "payload": logrecord})

Response: Forward log records to the frontend.


Instruction:
Write a Python 3 function that can
register a callback function that is run when idling for a given time span.

Response:
def _register_idle(self, idle_time, callback):
        """Register a callback function that is run when idling for a given
        time span (in seconds)."""
        self._idle_callback = callback
        self._idle_time = idle_time


Instruction:
Write a Python 3 function that can
update the internal status of the service object and notify frontend.

Response:
def __update_service_status(self, statuscode):
        """Set the internal status of the service object, and notify frontend."""
        if self.__service_status != statuscode:
            self.__service_status = statuscode
            self.__send_service_status_to_frontend()


Instruction:
Create a Python 3 function for
setting a new name for this service and notify the frontend accordingly.

Response:
def _set_name(self, name):
        """Set a new name for this service, and notify the frontend accordingly."""
        self._service_name = name
        self.__send_to_frontend({"band": "set_name", "name": self._service_name})


Instruction:
Implement a Python 3 function for
initializing the logging for the service process.

Response:
def initialize_logging(self):
        """Reset the logging for the service process. All logged messages are
        forwarded to the frontend. If any filtering is desired, then this must
        take place on the service side."""
        # Reset logging to pass logrecords into the queue to the frontend only.
        # Existing handlers may be broken as they were copied into a new process,
        # so should be discarded.
        for loggername in [None] + list(logging.Logger.manager.loggerDict.keys()):
            logger = logging.getLogger(loggername)
            while logger.handlers:
                logger.removeHandler(logger.handlers[0])

        # Re-enable logging to console
        root_logger = logging.getLogger()

        # By default pass all warning (and higher) level messages to the frontend
        root_logger.setLevel(logging.WARN)
        root_logger.addHandler(workflows.logging.CallbackHandler(self._log_send))

        # Set up the service logger and pass all info (and higher) level messages
        # (or other level if set differently)
        self.log = logging.getLogger(self._logger_name)

        if self.start_kwargs.get("verbose_log"):
            self.log_verbosity = logging.DEBUG
        self.log.setLevel(self.log_verbosity)

        # Additionally, write all critical messages directly to console
        console = logging.StreamHandler()
        console.setLevel(logging.CRITICAL)
        root_logger.addHandler(console)


Instruction:
Can you write a function in Python 3 where it
starts listening to command queue process commands in main loop

Response:
def start(self, **kwargs):
        """Start listening to command queue, process commands in main loop,
        set status, etc...
        This function is most likely called by the frontend in a separate
        process."""

        # Keep a copy of keyword arguments for use in subclasses
        self.start_kwargs.update(kwargs)
        try:
            self.initialize_logging()

            self.__update_service_status(self.SERVICE_STATUS_STARTING)

            self.start_transport()

            self.initializing()
            self._register("command", self.__process_command)

            if self.__pipe_commands is None:
                # can only listen to commands if command queue is defined
                self.__shutdown = True
            else:
                # start listening to command queue in separate thread
                self.__start_command_queue_listener()

            while not self.__shutdown:  # main loop
                self.__update_service_status(self.SERVICE_STATUS_IDLE)

                if self._idle_time is None:
                    task = self.__queue.get()
                else:
                    try:
                        task = self.__queue.get(True, self._idle_time)
                    except queue.Empty:
                        self.__update_service_status(self.SERVICE_STATUS_TIMER)
                        if self._idle_callback:
                            self._idle_callback()
                        continue

                self.__update_service_status(self.SERVICE_STATUS_PROCESSING)

                if task[0] == Priority.COMMAND:
                    message = task[2]
                    if message and "band" in message:
                        processor = self.__callback_register.get(message["band"])
                        if processor is None:
                            self.log.warning(
                                "received message on unregistered band\n%s", message
                            )
                        else:
                            processor(message.get("payload"))
                    else:
                        self.log.warning(
                            "received message without band information\n%s", message
                        )
                elif task[0] == Priority.TRANSPORT:
                    callback, header, message = task[2]
                    callback(header, message)
                else:
                    self.log.warning("Unknown item on main service queue\n%r", task)

        except KeyboardInterrupt:
            self.log.warning("Ctrl+C detected. Shutting down.")

        except Exception as e:
            self.process_uncaught_exception(e)
            self.__update_service_status(self.SERVICE_STATUS_ERROR)
            self.in_shutdown()
            return

        try:
            self.__update_service_status(self.SERVICE_STATUS_SHUTDOWN)
            self.in_shutdown()
            self.__update_service_status(self.SERVICE_STATUS_END)
        except Exception as e:
            self.process_uncaught_exception(e)
            self.__update_service_status(self.SERVICE_STATUS_ERROR)