Instruction:
Write a Python 3 script to
push a new data package to the storage.

Response:
def push_datapackage(descriptor, backend, **backend_options):
    """Push Data Package to storage.

    All parameters should be used as keyword arguments.

    Args:
        descriptor (str): path to descriptor
        backend (str): backend name like `sql` or `bigquery`
        backend_options (dict): backend options mentioned in backend docs

    """

    # Deprecated
    warnings.warn(
        'Functions "push/pull_datapackage" are deprecated. '
        'Please use "Package" class',
        UserWarning)

    # Init maps
    tables = []
    schemas = []
    datamap = {}
    mapping = {}

    # Init model
    model = Package(descriptor)

    # Get storage
    plugin = import_module('jsontableschema.plugins.%s' % backend)
    storage = plugin.Storage(**backend_options)

    # Collect tables/schemas/data
    for resource in model.resources:
        if not resource.tabular:
            continue
        name = resource.descriptor.get('name', None)
        table = _convert_path(resource.descriptor['path'], name)
        schema = resource.descriptor['schema']
        data = resource.table.iter(keyed=True)
        # TODO: review
        def values(schema, data):
            for item in data:
                row = []
                for field in schema['fields']:
                    row.append(item.get(field['name'], None))
                yield tuple(row)
        tables.append(table)
        schemas.append(schema)
        datamap[table] = values(schema, data)
        if name is not None:
            mapping[name] = table
    schemas = _convert_schemas(mapping, schemas)

    # Create tables
    for table in tables:
        if table in storage.buckets:
            storage.delete(table)
    storage.create(tables, schemas)

    # Write data to tables
    for table in storage.buckets:
        if table in datamap:
            storage.write(table, datamap[table])
    return storage


Instruction:
How would you code a function in Python 3 to
pull a datapackage from the storage.

Response:
def pull_datapackage(descriptor, name, backend, **backend_options):
    """Pull Data Package from storage.

    All parameters should be used as keyword arguments.

    Args:
        descriptor (str): path where to store descriptor
        name (str): name of the pulled datapackage
        backend (str): backend name like `sql` or `bigquery`
        backend_options (dict): backend options mentioned in backend docs

    """

    # Deprecated
    warnings.warn(
        'Functions "push/pull_datapackage" are deprecated. '
        'Please use "Package" class',
        UserWarning)

    # Save datapackage name
    datapackage_name = name

    # Get storage
    plugin = import_module('jsontableschema.plugins.%s' % backend)
    storage = plugin.Storage(**backend_options)

    # Iterate over tables
    resources = []
    for table in storage.buckets:

        # Prepare
        schema = storage.describe(table)
        base = os.path.dirname(descriptor)
        path, name = _restore_path(table)
        fullpath = os.path.join(base, path)

        # Write data
        helpers.ensure_dir(fullpath)
        with io.open(fullpath, 'wb') as file:
            model = Schema(deepcopy(schema))
            data = storage.iter(table)
            writer = csv.writer(file, encoding='utf-8')
            writer.writerow(model.headers)
            for row in data:
                writer.writerow(row)

        # Add resource
        resource = {'schema': schema, 'path': path}
        if name is not None:
            resource['name'] = name
        resources.append(resource)

    # Write descriptor
    mode = 'w'
    encoding = 'utf-8'
    if six.PY2:
        mode = 'wb'
        encoding = None
    resources = _restore_resources(resources)
    helpers.ensure_dir(descriptor)
    with io.open(descriptor,
                 mode=mode,
                 encoding=encoding) as file:
        descriptor = {
            'name': datapackage_name,
            'resources': resources,
        }
        json.dump(descriptor, file, indent=4)
    return storage


Instruction:
Given the following Python 3 function, write the documentation
def _convert_path(path, name):
    table = os.path.splitext(path)[0]
    table = table.replace(os.path.sep, '__')
    if name is not None:
        table = '___'.join([table, name])
    table = re.sub('[^0-9a-zA-Z_]+', '_', table)
    table = table.lower()
    return table

Response: Convert resource s path and name to storage s table name.


Instruction:
Can you implement a function in Python 3 that
restores resource s path and name from storage s table.

Response:
def _restore_path(table):
    """Restore resource's path and name from storage's table.

    Args:
        table (str): table name

    Returns:
        (str, str): resource path and name

    """
    name = None
    splited = table.split('___')
    path = splited[0]
    if len(splited) == 2:
        name = splited[1]
    path = path.replace('__', os.path.sep)
    path += '.csv'
    return path, name


Instruction:
Can you write a function in Python 3 where it
converts schemas to be compatible with storage schemas.

Response:
def _convert_schemas(mapping, schemas):
    """Convert schemas to be compatible with storage schemas.

    Foreign keys related operations.

    Args:
        mapping (dict): mapping between resource name and table name
        schemas (list): schemas

    Raises:
        ValueError: if there is no resource
            for some foreign key in given mapping

    Returns:
        list: converted schemas

    """
    schemas = deepcopy(schemas)
    for schema in schemas:
        for fk in schema.get('foreignKeys', []):
            resource = fk['reference']['resource']
            if resource != 'self':
                if resource not in mapping:
                    message = 'Not resource "%s" for foreign key "%s"'
                    message = message % (resource, fk)
                    raise ValueError(message)
                fk['reference']['resource'] = mapping[resource]
    return schemas


Instruction:
Create a Python 3 function to
restore schemas from being compatible with storage schemas.

Response:
def _restore_resources(resources):
    """Restore schemas from being compatible with storage schemas.

    Foreign keys related operations.

    Args:
        list: resources from storage

    Returns:
        list: restored resources

    """
    resources = deepcopy(resources)
    for resource in resources:
        schema = resource['schema']
        for fk in schema.get('foreignKeys', []):
            _, name = _restore_path(fk['reference']['resource'])
            fk['reference']['resource'] = name
    return resources


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _make_non_blocking(file_obj):

    if USING_WINDOWS:
        LPDWORD = POINTER(DWORD)
        PIPE_NOWAIT = wintypes.DWORD(0x00000001)

        SetNamedPipeHandleState = windll.kernel32.SetNamedPipeHandleState
        SetNamedPipeHandleState.argtypes = [HANDLE, LPDWORD, LPDWORD, LPDWORD]
        SetNamedPipeHandleState.restype = BOOL

        h = msvcrt.get_osfhandle(file_obj.fileno())

        res = windll.kernel32.SetNamedPipeHandleState(h, byref(PIPE_NOWAIT), None, None)
        if res == 0:
            raise ValueError(WinError())

    else:
        # Set the file status flag (F_SETFL) on the pipes to be non-blocking
        # so we can attempt to read from a pipe with no new data without locking
        # the program up
        fcntl.fcntl(file_obj, fcntl.F_SETFL, os.O_NONBLOCK)

Response: make file object non - blocking


Instruction:
Write a Python 3 script to
spawn a new gdb subprocess with the arguments supplied to the object during initialization.

Response:
def spawn_new_gdb_subprocess(self):
        """Spawn a new gdb subprocess with the arguments supplied to the object
        during initialization. If gdb subprocess already exists, terminate it before
        spanwing a new one.
        Return int: gdb process id
        """
        if self.gdb_process:
            self.logger.debug(
                "Killing current gdb subprocess (pid %d)" % self.gdb_process.pid
            )
            self.exit()

        self.logger.debug('Launching gdb: "%s"' % " ".join(self.cmd))

        # Use pipes to the standard streams
        self.gdb_process = subprocess.Popen(
            self.cmd,
            shell=False,
            stdout=subprocess.PIPE,
            stdin=subprocess.PIPE,
            stderr=subprocess.PIPE,
            bufsize=0,
        )

        _make_non_blocking(self.gdb_process.stdout)
        _make_non_blocking(self.gdb_process.stderr)

        # save file numbers for use later
        self.stdout_fileno = self.gdb_process.stdout.fileno()
        self.stderr_fileno = self.gdb_process.stderr.fileno()
        self.stdin_fileno = self.gdb_process.stdin.fileno()

        self.read_list = [self.stdout_fileno, self.stderr_fileno]
        self.write_list = [self.stdin_fileno]

        # string buffers for unifinished gdb output
        self._incomplete_output = {"stdout": None, "stderr": None}
        return self.gdb_process.pid


Instruction:
Implement a Python 3 function for
verifying that the gdb process is still running. Raise NoGdbProcessError if either of the above are not true.

Response:
def verify_valid_gdb_subprocess(self):
        """Verify there is a process object, and that it is still running.
        Raise NoGdbProcessError if either of the above are not true."""
        if not self.gdb_process:
            raise NoGdbProcessError("gdb process is not attached")

        elif self.gdb_process.poll() is not None:
            raise NoGdbProcessError(
                "gdb process has already finished with return code: %s"
                % str(self.gdb_process.poll())
            )


Instruction:
Can you write a function in Python 3 where it
writes to gdb process.

Response:
def write(
        self,
        mi_cmd_to_write,
        timeout_sec=DEFAULT_GDB_TIMEOUT_SEC,
        raise_error_on_timeout=True,
        read_response=True,
    ):
        """Write to gdb process. Block while parsing responses from gdb for a maximum of timeout_sec.

        Args:
            mi_cmd_to_write (str or list): String to write to gdb. If list, it is joined by newlines.
            timeout_sec (float): Maximum number of seconds to wait for response before exiting. Must be >= 0.
            raise_error_on_timeout (bool): If read_response is True, raise error if no response is received
            read_response (bool): Block and read response. If there is a separate thread running,
            this can be false, and the reading thread read the output.
        Returns:
            List of parsed gdb responses if read_response is True, otherwise []
        Raises:
            NoGdbProcessError if there is no gdb subprocess running
            TypeError if mi_cmd_to_write is not valid
        """
        self.verify_valid_gdb_subprocess()
        if timeout_sec < 0:
            self.logger.warning("timeout_sec was negative, replacing with 0")
            timeout_sec = 0

        # Ensure proper type of the mi command
        if type(mi_cmd_to_write) in [str, unicode]:
            pass
        elif type(mi_cmd_to_write) == list:
            mi_cmd_to_write = "\n".join(mi_cmd_to_write)
        else:
            raise TypeError(
                "The gdb mi command must a be str or list. Got "
                + str(type(mi_cmd_to_write))
            )

        self.logger.debug("writing: %s", mi_cmd_to_write)

        if not mi_cmd_to_write.endswith("\n"):
            mi_cmd_to_write_nl = mi_cmd_to_write + "\n"
        else:
            mi_cmd_to_write_nl = mi_cmd_to_write

        if USING_WINDOWS:
            # select not implemented in windows for pipes
            # assume it's always ready
            outputready = [self.stdin_fileno]
        else:
            _, outputready, _ = select.select([], self.write_list, [], timeout_sec)
        for fileno in outputready:
            if fileno == self.stdin_fileno:
                # ready to write
                self.gdb_process.stdin.write(mi_cmd_to_write_nl.encode())
                # don't forget to flush for Python3, otherwise gdb won't realize there is data
                # to evaluate, and we won't get a response
                self.gdb_process.stdin.flush()
            else:
                self.logger.error("got unexpected fileno %d" % fileno)

        if read_response is True:
            return self.get_gdb_response(
                timeout_sec=timeout_sec, raise_error_on_timeout=raise_error_on_timeout
            )

        else:
            return []


Instruction:
Here you have a function in Python 3, explain what it does
def get_gdb_response(
        self, timeout_sec=DEFAULT_GDB_TIMEOUT_SEC, raise_error_on_timeout=True
    ):
        """Get response from GDB, and block while doing so. If GDB does not have any response ready to be read
        by timeout_sec, an exception is raised.

        Args:
            timeout_sec (float): Maximum time to wait for reponse. Must be >= 0. Will return after
            raise_error_on_timeout (bool): Whether an exception should be raised if no response was found
            after timeout_sec

        Returns:
            List of parsed GDB responses, returned from gdbmiparser.parse_response, with the
            additional key 'stream' which is either 'stdout' or 'stderr'

        Raises:
            GdbTimeoutError if response is not received within timeout_sec
            ValueError if select returned unexpected file number
            NoGdbProcessError if there is no gdb subprocess running
        """

        self.verify_valid_gdb_subprocess()
        if timeout_sec < 0:
            self.logger.warning("timeout_sec was negative, replacing with 0")
            timeout_sec = 0

        if USING_WINDOWS:
            retval = self._get_responses_windows(timeout_sec)
        else:
            retval = self._get_responses_unix(timeout_sec)

        if not retval and raise_error_on_timeout:
            raise GdbTimeoutError(
                "Did not get response from gdb after %s seconds" % timeout_sec
            )

        else:
            return retval

Response: Get response from GDB and block while doing so.


Instruction:
Write a Python 3 function that can
get responses on windows.

Response:
def _get_responses_windows(self, timeout_sec):
        """Get responses on windows. Assume no support for select and use a while loop."""
        timeout_time_sec = time.time() + timeout_sec
        responses = []
        while True:
            try:
                self.gdb_process.stdout.flush()
                if PYTHON3:
                    raw_output = self.gdb_process.stdout.readline().replace(
                        b"\r", b"\n"
                    )
                else:
                    raw_output = self.gdb_process.stdout.read().replace(b"\r", b"\n")
                responses += self._get_responses_list(raw_output, "stdout")
            except IOError:
                pass

            try:
                self.gdb_process.stderr.flush()
                if PYTHON3:
                    raw_output = self.gdb_process.stderr.readline().replace(
                        b"\r", b"\n"
                    )
                else:
                    raw_output = self.gdb_process.stderr.read().replace(b"\r", b"\n")
                responses += self._get_responses_list(raw_output, "stderr")
            except IOError:
                pass

            if time.time() > timeout_time_sec:
                break

        return responses


Instruction:
Explain what the following Python 3 code does
def _get_responses_unix(self, timeout_sec):
        timeout_time_sec = time.time() + timeout_sec
        responses = []
        while True:
            select_timeout = timeout_time_sec - time.time()
            # I prefer to not pass a negative value to select
            if select_timeout <= 0:
                select_timeout = 0
            events, _, _ = select.select(self.read_list, [], [], select_timeout)
            responses_list = None  # to avoid infinite loop if using Python 2
            try:
                for fileno in events:
                    # new data is ready to read
                    if fileno == self.stdout_fileno:
                        self.gdb_process.stdout.flush()
                        raw_output = self.gdb_process.stdout.read()
                        stream = "stdout"

                    elif fileno == self.stderr_fileno:
                        self.gdb_process.stderr.flush()
                        raw_output = self.gdb_process.stderr.read()
                        stream = "stderr"

                    else:
                        raise ValueError(
                            "Developer error. Got unexpected file number %d" % fileno
                        )

                    responses_list = self._get_responses_list(raw_output, stream)
                    responses += responses_list

            except IOError:  # only occurs in python 2.7
                pass

            if timeout_sec == 0:  # just exit immediately
                break

            elif responses_list and self._allow_overwrite_timeout_times:
                # update timeout time to potentially be closer to now to avoid lengthy wait times when nothing is being output by gdb
                timeout_time_sec = min(
                    time.time() + self.time_to_check_for_additional_output_sec,
                    timeout_time_sec,
                )

            elif time.time() > timeout_time_sec:
                break

        return responses

Response: Get the list of responses on unix - like system. Use select to wait for output.


Instruction:
Write a Python 3 function for
getting parsed response list from string output

Response:
def _get_responses_list(self, raw_output, stream):
        """Get parsed response list from string output
        Args:
            raw_output (unicode): gdb output to parse
            stream (str): either stdout or stderr
        """
        responses = []

        raw_output, self._incomplete_output[stream] = _buffer_incomplete_responses(
            raw_output, self._incomplete_output.get(stream)
        )

        if not raw_output:
            return responses

        response_list = list(
            filter(lambda x: x, raw_output.decode(errors="replace").split("\n"))
        )  # remove blank lines

        # parse each response from gdb into a dict, and store in a list
        for response in response_list:
            if gdbmiparser.response_is_finished(response):
                pass
            else:
                parsed_response = gdbmiparser.parse_response(response)
                parsed_response["stream"] = stream

                self.logger.debug("%s", pformat(parsed_response))

                responses.append(parsed_response)

        return responses


Instruction:
Explain what the following Python 3 code does
def send_signal_to_gdb(self, signal_input):
        try:
            signal = int(signal_input)
        except Exception:
            signal = SIGNAL_NAME_TO_NUM.get(signal_input.upper())

        if not signal:
            raise ValueError(
                'Could not find signal corresponding to "%s"' % str(signal)
            )

        if self.gdb_process:
            os.kill(self.gdb_process.pid, signal)
        else:
            raise NoGdbProcessError(
                "Cannot send signal to gdb process because no process exists."
            )

Response: Send a signal to the gdb process.


Instruction:
Make a summary of the following Python 3 code
def exit(self):
        if self.gdb_process:
            self.gdb_process.terminate()
            self.gdb_process.communicate()
        self.gdb_process = None
        return None

Response:
Terminate gdb process
            Returns None


Instruction:
Write a Python 3 function for
building and debug an application programatically For a list of GDB MI commands, see https://www.sourceware.org/gdb/onlinedocs/gdb/GDB_002fMI.html

Response:
def main(verbose=True):
    """Build and debug an application programatically

    For a list of GDB MI commands, see https://www.sourceware.org/gdb/onlinedocs/gdb/GDB_002fMI.html
    """

    # Build C program
    find_executable(MAKE_CMD)
    if not find_executable(MAKE_CMD):
        print(
            'Could not find executable "%s". Ensure it is installed and on your $PATH.'
            % MAKE_CMD
        )
        exit(1)
    subprocess.check_output([MAKE_CMD, "-C", SAMPLE_C_CODE_DIR, "--quiet"])

    # Initialize object that manages gdb subprocess
    gdbmi = GdbController(verbose=verbose)

    # Send gdb commands. Gdb machine interface commands are easier to script around,
    # hence the name "machine interface".
    # Responses are automatically printed as they are received if verbose is True.
    # Responses are returned after writing, by default.

    # Load the file
    responses = gdbmi.write("-file-exec-and-symbols %s" % SAMPLE_C_BINARY)
    # Get list of source files used to compile the binary
    responses = gdbmi.write("-file-list-exec-source-files")
    # Add breakpoint
    responses = gdbmi.write("-break-insert main")
    # Run
    responses = gdbmi.write("-exec-run")
    responses = gdbmi.write("-exec-next")
    responses = gdbmi.write("-exec-next")
    responses = gdbmi.write("-exec-continue")  # noqa: F841

    # gdbmi.gdb_process will be None because the gdb subprocess (and its inferior
    # program) will be terminated
    gdbmi.exit()


Instruction:
How would you explain what the following Python 3 function does
def read(self, count):
        new_index = self.index + count
        if new_index > self.len:
            buf = self.raw_text[self.index :]  # return to the end, don't fail
        else:
            buf = self.raw_text[self.index : new_index]
        self.index = new_index

        return buf

Response: Read count characters starting at self. index and return those characters as a string


Instruction:
Write a Python 3 script to
advance the index past specific chars

Response:
def advance_past_chars(self, chars):
        """Advance the index past specific chars
        Args chars (list): list of characters to advance past

        Return substring that was advanced past
        """
        start_index = self.index
        while True:
            current_char = self.raw_text[self.index]
            self.index += 1
            if current_char in chars:
                break

            elif self.index == self.len:
                break

        return self.raw_text[start_index : self.index - 1]


Instruction:
Create a Python 3 function for
advancing to the next string that has gdb escapes.

Response:
def advance_past_string_with_gdb_escapes(self, chars_to_remove_gdb_escape=None):
        """characters that gdb escapes that should not be
        escaped by this parser
        """

        if chars_to_remove_gdb_escape is None:
            chars_to_remove_gdb_escape = ['"']

        buf = ""
        while True:
            c = self.raw_text[self.index]
            self.index += 1
            logging.debug("%s", fmt_cyan(c))

            if c == "\\":
                # We are on a backslash and there is another character after the backslash
                # to parse. Handle this case specially since gdb escaped it for us

                # Get the next char that is being escaped
                c2 = self.raw_text[self.index]
                self.index += 1
                # only store the escaped character in the buffer; don't store the backslash
                # (don't leave it escaped)
                buf += c2

            elif c == '"':
                # Quote is closed. Exit (and don't include the end quote).
                break

            else:
                # capture this character, and keep capturing
                buf += c
        return buf


Instruction:
Make a summary of the following Python 3 code
def parse_response(gdb_mi_text):
    stream = StringStream(gdb_mi_text, debug=_DEBUG)

    if _GDB_MI_NOTIFY_RE.match(gdb_mi_text):
        token, message, payload = _get_notify_msg_and_payload(gdb_mi_text, stream)
        return {
            "type": "notify",
            "message": message,
            "payload": payload,
            "token": token,
        }

    elif _GDB_MI_RESULT_RE.match(gdb_mi_text):
        token, message, payload = _get_result_msg_and_payload(gdb_mi_text, stream)
        return {
            "type": "result",
            "message": message,
            "payload": payload,
            "token": token,
        }

    elif _GDB_MI_CONSOLE_RE.match(gdb_mi_text):
        return {
            "type": "console",
            "message": None,
            "payload": _GDB_MI_CONSOLE_RE.match(gdb_mi_text).groups()[0],
        }

    elif _GDB_MI_LOG_RE.match(gdb_mi_text):
        return {
            "type": "log",
            "message": None,
            "payload": _GDB_MI_LOG_RE.match(gdb_mi_text).groups()[0],
        }

    elif _GDB_MI_TARGET_OUTPUT_RE.match(gdb_mi_text):
        return {
            "type": "target",
            "message": None,
            "payload": _GDB_MI_TARGET_OUTPUT_RE.match(gdb_mi_text).groups()[0],
        }

    elif response_is_finished(gdb_mi_text):
        return {"type": "done", "message": None, "payload": None}

    else:
        # This was not gdb mi output, so it must have just been printed by
        # the inferior program that's being debugged
        return {"type": "output", "message": None, "payload": gdb_mi_text}

Response: Parse the gdb mi output into a dictionary.


Instruction:
Can you tell what is the following Python 3 function doing
def assert_match(actual_char_or_str, expected_char_or_str):
    if expected_char_or_str != actual_char_or_str:
        print("Expected")
        pprint(expected_char_or_str)
        print("")
        print("Got")
        pprint(actual_char_or_str)
        raise ValueError()

Response: Assert that the character or string of the current node matches the expected character.


Instruction:
How would you explain what the following Python 3 function does
def _get_notify_msg_and_payload(result, stream):
    token = stream.advance_past_chars(["=", "*"])
    token = int(token) if token != "" else None
    logger.debug("%s", fmt_green("parsing message"))
    message = stream.advance_past_chars([","])

    logger.debug("parsed message")
    logger.debug("%s", fmt_green(message))

    payload = _parse_dict(stream)
    return token, message.strip(), payload

Response: Get notify message and payload dict from stream.


Instruction:
Implement a Python 3 function for
getting result message and payload dict

Response:
def _get_result_msg_and_payload(result, stream):
    """Get result message and payload dict"""

    groups = _GDB_MI_RESULT_RE.match(result).groups()
    token = int(groups[0]) if groups[0] != "" else None
    message = groups[1]

    if groups[2] is None:
        payload = None
    else:
        stream.advance_past_chars([","])
        payload = _parse_dict(stream)

    return token, message, payload


Instruction:
Write a Python 3 script for
parsing a dictionary of key value pairs.

Response:
def _parse_dict(stream):
    """Parse dictionary, with optional starting character '{'
    return (tuple):
        Number of characters parsed from to_parse
        Parsed dictionary
    """
    obj = {}

    logger.debug("%s", fmt_green("parsing dict"))

    while True:
        c = stream.read(1)
        if c in _WHITESPACE:
            pass
        elif c in ["{", ","]:
            pass
        elif c in ["}", ""]:
            # end of object, exit loop
            break

        else:
            stream.seek(-1)
            key, val = _parse_key_val(stream)
            if key in obj:
                # This is a gdb bug. We should never get repeated keys in a dict!
                # See https://sourceware.org/bugzilla/show_bug.cgi?id=22217
                # and https://github.com/cs01/pygdbmi/issues/19
                # Example:
                #   thread-ids={thread-id="1",thread-id="2"}
                # Results in:
                #   thread-ids: {{'thread-id': ['1', '2']}}
                # Rather than the lossy
                #   thread-ids: {'thread-id': 2}  # '1' got overwritten!
                if isinstance(obj[key], list):
                    obj[key].append(val)
                else:
                    obj[key] = [obj[key], val]
            else:
                obj[key] = val

            look_ahead_for_garbage = True
            c = stream.read(1)
            while look_ahead_for_garbage:
                if c in ["}", ",", ""]:
                    look_ahead_for_garbage = False
                else:
                    # got some garbage text, skip it. for example:
                    # name="gdb"gargage  # skip over 'garbage'
                    # name="gdb"\n  # skip over '\n'
                    logger.debug("skipping unexpected charcter: " + c)
                    c = stream.read(1)
            stream.seek(-1)

    logger.debug("parsed dict")
    logger.debug("%s", fmt_green(obj))
    return obj


Instruction:
Can you implement a function in Python 3 that
parses key and value combination

Response:
def _parse_key_val(stream):
    """Parse key, value combination
    return (tuple):
        Parsed key (string)
        Parsed value (either a string, array, or dict)
    """

    logger.debug("parsing key/val")
    key = _parse_key(stream)
    val = _parse_val(stream)

    logger.debug("parsed key/val")
    logger.debug("%s", fmt_green(key))
    logger.debug("%s", fmt_green(val))

    return key, val


Instruction:
Explain what the following Python 3 code does
def _parse_key(stream):
    logger.debug("parsing key")

    key = stream.advance_past_chars(["="])

    logger.debug("parsed key:")
    logger.debug("%s", fmt_green(key))
    return key

Response: Parse a key value combination returning a Parsed key


Instruction:
Can you tell what is the following Python 3 function doing
def _parse_val(stream):

    logger.debug("parsing value")

    while True:
        c = stream.read(1)

        if c == "{":
            # Start object
            val = _parse_dict(stream)
            break

        elif c == "[":
            # Start of an array
            val = _parse_array(stream)
            break

        elif c == '"':
            # Start of a string
            val = stream.advance_past_string_with_gdb_escapes()
            break

        elif _DEBUG:
            raise ValueError("unexpected character: %s" % c)

        else:
            print(
                'pygdbmi warning: encountered unexpected character: "%s". Continuing.'
                % c
            )
            val = ""  # this will be overwritten if there are more characters to be read

    logger.debug("parsed value:")
    logger.debug("%s", fmt_green(val))

    return val

Response:
Parse a value from string
    returns a parsed value


Instruction:
Can you create a Python 3 function that
parses an array of items from a GDBMI stream.

Response:
def _parse_array(stream):
    """Parse an array, stream should be passed the initial [
    returns:
        Parsed array
    """

    logger.debug("parsing array")
    arr = []
    while True:
        c = stream.read(1)

        if c in _GDB_MI_VALUE_START_CHARS:
            stream.seek(-1)
            val = _parse_val(stream)
            arr.append(val)
        elif c in _WHITESPACE:
            pass
        elif c == ",":
            pass
        elif c == "]":
            # Stop when this array has finished. Note
            # that elements of this array can be also be arrays.
            break

    logger.debug("parsed array:")
    logger.debug("%s", fmt_green(arr))
    return arr


Instruction:
How would you explain what the following Python 3 function does
def as_parameters(*parameters, variables=None):
        s = json.dumps(parameters)
        s = s[1:-1]
        if variables:
            for v in variables:
                if v in parameters:
                    s = s.replace('"' + v + '"', v)
        return s

Response: Dump python list as the parameter of javascript function


Instruction:
How would you explain what the following Python 3 function does
def generate_local_url(self, js_name):
        host = self._settings['local_host'].format(**self._host_context).rstrip('/')
        return '{}/{}.js'.format(host, js_name)

Response: Generate the local url for a js file.


Instruction:
How would you code a function in Python 3 to
return a single item from an iterable with the given key.

Response:
def ifetch_single(iterable, key, default=EMPTY, getter=None):
    """
    getter() g(item, key):pass
    """

    def _getter(item):
        if getter:
            custom_getter = partial(getter, key=key)
            return custom_getter(item)
        else:
            try:
                attrgetter = operator.attrgetter(key)
                return attrgetter(item)
            except AttributeError:
                pass

            try:
                itemgetter = operator.itemgetter(key)
                return itemgetter(item)
            except KeyError:
                pass

            if default is not EMPTY:
                return default

            raise ValueError('Item %r has no attr or key for %r' % (item, key))

    return map(_getter, iterable)


Instruction:
Explain what the following Python 3 code does
def _get_or_create_subscription(self):
        topic_path = self._get_topic_path()
        subscription_name = '{}-{}-{}-worker'.format(
            queue.PUBSUB_OBJECT_PREFIX, self.name, uuid4().hex)
        subscription_path = self.subscriber_client.subscription_path(
            self.project, subscription_name)

        try:
            self.subscriber_client.get_subscription(subscription_path)
        except google.cloud.exceptions.NotFound:
            logger.info("Creating worker subscription {}".format(
                subscription_name))
            self.subscriber_client.create_subscription(
                subscription_path, topic_path)

        return subscription_path

Response: In a broadcast queue workers have a unique subscription ensuring that every worker recieves a copy of every task.


Instruction:
Can you generate the documentation for the following Python 3 function
def cleanup(self):
        if self.subscription:
            logger.info("Deleting worker subscription...")
            self.subscriber_client.delete_subscription(self.subscription)

Response: Deletes this worker s subscription.


Instruction:
Here you have a function in Python 3, explain what it does
def _get_or_create_subscription(self):
        topic_path = self._get_topic_path()
        subscription_name = '{}-{}-shared'.format(
            PUBSUB_OBJECT_PREFIX, self.name)
        subscription_path = self.subscriber_client.subscription_path(
            self.project, subscription_name)

        try:
            self.subscriber_client.get_subscription(subscription_path)
        except google.cloud.exceptions.NotFound:
            logger.info("Creating shared subscription {}".format(
                subscription_name))
            try:
                self.subscriber_client.create_subscription(
                    subscription_path, topic=topic_path)
            except google.cloud.exceptions.Conflict:
                # Another worker created the subscription before us, ignore.
                pass

        return subscription_path

Response: Get or create a shared subscription.


Instruction:
Can you generate the documentation for the following Python 3 function
def enqueue_task(self, task):
        data = dumps(task)

        if self._async:
            self.publisher_client.publish(self.topic_path, data=data)
            logger.info('Task {} queued.'.format(task.id))
        else:
            unpickled_task = unpickle(data)
            logger.info(
                'Executing task {} synchronously.'.format(unpickled_task.id)
            )
            with measure_time() as summary, self.queue_context():
                unpickled_task.execute(queue=self)
                summary(unpickled_task.summary())

        return TaskResult(task.id, self)

Response: Enqueues a task asynchronously.


Instruction:
Here you have a function in Python 3, explain what it does
def unpickle(pickled_string):
    try:
        obj = loads(pickled_string)
    except Exception as e:
        raise UnpickleError('Could not unpickle', pickled_string, e)
    return obj

Response: Unpickles a string and raises a unified UnpickleError in case anything fails.


Instruction:
How would you implement a function in Python 3 that
returns the result of the task.

Response:
def result(self, timeout=None):
        """Gets the result of the task.

        Arguments:
            timeout: Maximum seconds to wait for a result before raising a
                TimeoutError. If set to None, this will wait forever. If the
                queue doesn't store results and timeout is None, this call will
                never return.
        """
        start = time.time()
        while True:
            task = self.get_task()
            if not task or task.status not in (FINISHED, FAILED):
                if not timeout:
                    continue
                elif time.time() - start < timeout:
                    continue
                else:
                    raise TimeoutError()

            if task.status == FAILED:
                raise task.result

            return task.result


Instruction:
Write a Python 3 function for
launching a Process, return his pid

Response:
def service_start(service=None, param=None):
    """
        Launch a Process, return his pid
    """
    if service is not None:
        to_run = ["python", service]
        if param is not None:
            to_run += param
        return subprocess.Popen(to_run)
    return False


Instruction:
How would you code a function in Python 3 to
update the running processes list and return the list of the running processes

Response:
def update_running_pids(old_procs):
    """
        Update the list of the running process and return the list
    """
    new_procs = []
    for proc in old_procs:
        if proc.poll() is None and check_pid(proc.pid):
            publisher.debug(str(proc.pid) + ' is alive')
            new_procs.append(proc)
        else:
            try:
                publisher.debug(str(proc.pid) + ' is gone')
                os.kill(proc.pid, signal.SIGKILL)
            except:
                # the process is just already gone
                pass
    return new_procs


Instruction:
Write a Python 3 function for
running the split_processing function for the RIPE process_name and filenames.

Response:
def run_splitted_processing(max_simultaneous_processes, process_name,
                            filenames):
    """
        Run processes which push the routing dump of the RIPE in a redis
        database.
        The dump has been splitted in multiple files and each process run
        on one of this files.
    """
    pids = []
    while len(filenames) > 0:
        while len(filenames) > 0 and len(pids) < max_simultaneous_processes:
            filename = filenames.pop()
            pids.append(service_start(service=process_name,
                                      param=['-f', filename, '-d',
                                             imported_day]))
        while len(pids) == max_simultaneous_processes:
            time.sleep(sleep_timer)
            pids = update_running_pids(pids)
    while len(pids) > 0:
        # Wait until all the processes are finished
        time.sleep(sleep_timer)
        pids = update_running_pids(pids)


Instruction:
Can you write a function in Python 3 where it
splits the file into several files.

Response:
def fsplit(file_to_split):
    """
        Split the file and return the list of filenames.
    """
    dirname = file_to_split + '_splitted'
    if not os.path.exists(dirname):
        os.mkdir(dirname)
    part_file_size = os.path.getsize(file_to_split) / number_of_files + 1
    splitted_files = []
    with open(file_to_split, "r") as f:
        number = 0
        actual = 0
        while 1:
            prec = actual
            # Jump of "size" from the current place in the file
            f.seek(part_file_size, os.SEEK_CUR)

            # find the next separator or EOF
            s = f.readline()
            if len(s) == 0:
                s = f.readline()
            while len(s) != 0 and s != separator:
                s = f.readline()

            # Get the current place
            actual = f.tell()
            new_file = os.path.join(dirname, str(number))

            # Create the new file
            with open(file_to_split, "r") as temp:
                temp.seek(prec)
                # Get the text we want to put in the new file
                copy = temp.read(actual - prec)
                # Write the new file
                open(new_file, 'w').write(copy)
            splitted_files.append(new_file)
            number += 1

            # End of file
            if len(s) == 0:
                break
    return splitted_files


Instruction:
Here you have a function in Python 3, explain what it does
def __entry_point():
    ip = request.remote_addr
    ua = request.headers.get('User-Agent', 'Empty User-Agent')
    method = request.json.get('method')
    if method is None:
        __query_logging(ip, ua, method, level='warning')
        return json.dumps({'error': 'No method provided.'})
    if method not in authorized_methods:
        # unauthorized query
        __query_logging(ip, ua, method, level='warning')
        return json.dumps({'error': 'Unauthorized method.'})
    fct = globals().get(method)
    if fct is None:
        # unknown method, the method is authorized, but does not exists...
        __query_logging(ip, ua, method, level='warning')
        return json.dumps({'error': 'Unknown method.'})
    if request.json.get('ip') is None:
        __query_logging(ip, ua, method, level='warning')
        return json.dumps({'error': 'No IP provided, not going to work.'})
    try:
        result = fct(request.json)
        __query_logging(ip, ua, method, request.json.get('ip'),
                        request.json.get('announce_date'), request.json.get('days_limit'))
        return result
    except Exception:
        __query_logging(ip, ua, method, request.json.get('ip'), level='error')
        return json.dumps({'error': 'Something went wrong.'})

Response: This function is called by the entry point of the API. It is called by the API when a request is made on the URL.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def asn(self, ip, announce_date=None):
        assignations, announce_date, _ = self.run(ip, announce_date)
        return next((assign for assign in assignations if assign is not None), None), announce_date

Response: Get the ASN. nacute entry for an IP.


Instruction:
Explain what the following Python 3 code does
def date_asn_block(self, ip, announce_date=None):
        assignations, announce_date, keys = self.run(ip, announce_date)
        pos = next((i for i, j in enumerate(assignations) if j is not None), None)
        if pos is not None:
            block = keys[pos]
            if block != '0.0.0.0/0':
                return announce_date, assignations[pos], block
        return None

Response: Returns the ASN and the IP Block announcing the IP at a specific date.


Instruction:
Can you generate the documentation for the following Python 3 function
def history(self, ip, days_limit=None):
        all_dates = sorted(self.routing_db.smembers('imported_dates'), reverse=True)
        if days_limit is not None:
            all_dates = all_dates[:days_limit]
        return [self.date_asn_block(ip, date) for date in all_dates]

Response: Get the full history of an IP. It takes time.


Instruction:
Explain what the following Python 3 code does
def aggregate_history(self, ip, days_limit=None):
        first_date = None
        last_date = None
        prec_asn = None
        prec_block = None
        for entry in self.history(ip, days_limit):
            if entry is None:
                continue
            date, asn, block = entry
            if first_date is None:
                last_date = date
                first_date = date
                prec_asn = asn
                prec_block = block
            elif prec_asn == asn and prec_block == block:
                first_date = date
            else:
                yield first_date, last_date, prec_asn, prec_block
                last_date = date
                first_date = date
                prec_asn = asn
                prec_block = block
        if first_date is not None:
            yield first_date, last_date, prec_asn, prec_block

Response: Get the full history of an IP and aggregate it for one day per day.


Instruction:
Can you create a Python 3 function that
downloads a URL to a file in the bview directory.

Response:
def downloadURL(url, filename):
    """
        Inconditianilly download the URL in a temporary directory.
        When finished, the file is moved in the real directory.
        Like this an other process will not attempt to extract an inclomplete file.
    """
    path_temp_bviewfile = os.path.join(c.raw_data, c.bview_dir, 'tmp', filename)
    path_bviewfile = os.path.join(c.raw_data, c.bview_dir, filename)
    try:
        f = urlopen(url)
    except:
        return False
    if f.getcode() != 200:
        publisher.warning('{} unavailable, code: {}'.format(url, f.getcode()))
        return False
    try:
        with open(path_temp_bviewfile, 'w') as outfile:
            outfile.write(f.read())
        os.rename(path_temp_bviewfile, path_bviewfile)
    except:
        os.remove(path_temp_bviewfile)
        return False
    return True


Instruction:
Create a Python 3 function for
verifying that the file has not already been downloaded.

Response:
def already_downloaded(filename):
    """
        Verify that the file has not already been downloaded.
    """
    cur_file = os.path.join(c.bview_dir, filename)
    old_file = os.path.join(c.bview_dir, 'old', filename)
    if not os.path.exists(cur_file) and not os.path.exists(old_file):
        return False
    return True


Instruction:
How would you implement a function in Python 3 that
builds interval of urls to download.

Response:
def to_download():
    """
        Build interval of urls to download.
        We always get the first file of the next day.
        Ex: 2013-01-01 => 2013-01-02.0000
    """
    first_day = parse(interval_first)
    last_day = parse(interval_last)
    format_change = parse('2010-06-14')
    one_day = datetime.timedelta(1)
    cur_day = first_day
    url_list = []
    while cur_day < last_day:
        fname = filename.format(day=cur_day.strftime("%Y%m%d"))
        if cur_day > format_change:
            cur_day += one_day
            url = base_url.format(year_month=cur_day.strftime("%Y.%m"),
                                  file_day=cur_day.strftime("%Y%m%d"))
        else:
            url = base_url_old.format(year_month=cur_day.strftime("%Y.%m"),
                                      file_day=cur_day.strftime("%Y%m%d"))
            cur_day += one_day
        url_list.append((fname, url))
    return sorted(url_list, key=lambda tup: tup[0], reverse=True)


Instruction:
Here you have a function in Python 3, explain what it does
def get_page_url(page_num, current_app, url_view_name, url_extra_args, url_extra_kwargs, url_param_name, url_get_params, url_anchor):
    if url_view_name is not None:
        # Add page param to the kwargs list. Overrides any previously set parameter of the same name.
        url_extra_kwargs[url_param_name] = page_num

        try:
            url = reverse(url_view_name, args=url_extra_args, kwargs=url_extra_kwargs, current_app=current_app)
        except NoReverseMatch as e:  # Attempt to load view from application root, allowing the use of non-namespaced view names if your view is defined in the root application
            if settings.SETTINGS_MODULE:

                if django.VERSION < (1, 9, 0):
                    separator  = '.'
                else:
                    separator  = ':' # Namespace separator changed to colon after 1.8

                project_name = settings.SETTINGS_MODULE.split('.')[0]
                try:
                    url = reverse(project_name + separator + url_view_name, args=url_extra_args, kwargs=url_extra_kwargs, current_app=current_app)
                except NoReverseMatch:
                    raise e # Raise the original exception so the error message doesn't confusingly include something the Developer didn't add to the view name themselves
            else:
                raise e # We can't determine the project name so just re-throw the exception

    else:
        url = ''
        url_get_params = url_get_params or QueryDict(url)
        url_get_params = url_get_params.copy()
        url_get_params[url_param_name] = str(page_num)

    if len(url_get_params) > 0:
        if not isinstance(url_get_params, QueryDict):
            tmp = QueryDict(mutable=True)
            tmp.update(url_get_params)
            url_get_params = tmp
        url += '?' + url_get_params.urlencode()

    if (url_anchor is not None):
        url += '#' + url_anchor

    return url

Response: Helper function to return a valid URL string given the template tag parameters and the parameters of the page.


Instruction:
Here you have a function in Python 3, explain what it does
def bootstrap_paginate(parser, token):
    bits = token.split_contents()
    if len(bits) < 2:
        raise TemplateSyntaxError("'%s' takes at least one argument"
                                  " (Page object reference)" % bits[0])
    page = parser.compile_filter(bits[1])
    kwargs = {}
    bits = bits[2:]

    kwarg_re = re.compile(r'(\w+)=(.+)')

    if len(bits):
        for bit in bits:
            match = kwarg_re.match(bit)
            if not match:
                raise TemplateSyntaxError("Malformed arguments to bootstrap_pagination paginate tag")
            name, value = match.groups()
            kwargs[name] = parser.compile_filter(value)

    return BootstrapPaginationNode(page, kwargs)

Response: Renders a page object as a Twitter Bootstrap styled pagination bar.


Instruction:
Can you create a Python 3 function that
configures all Jenkins devel jobs.

Response:
def configure_devel_jobs(
        config_url, rosdistro_name, source_build_name, groovy_script=None,
        dry_run=False, whitelist_repository_names=None):
    """
    Configure all Jenkins devel jobs.

    L{configure_release_job} will be invoked for source repository and target
    which matches the build file criteria.
    """
    config = get_config_index(config_url)
    build_files = get_source_build_files(config, rosdistro_name)
    build_file = build_files[source_build_name]

    index = get_index(config.rosdistro_index_url)

    dist_cache = None
    if build_file.notify_maintainers:
        dist_cache = get_distribution_cache(index, rosdistro_name)

    # get targets
    targets = []
    for os_name in build_file.targets.keys():
        for os_code_name in build_file.targets[os_name].keys():
            for arch in build_file.targets[os_name][os_code_name]:
                targets.append((os_name, os_code_name, arch))
    print('The build file contains the following targets:')
    for os_name, os_code_name, arch in targets:
        print('  -', os_name, os_code_name, arch)

    dist_file = get_distribution_file(index, rosdistro_name, build_file)
    if not dist_file:
        print('No distribution file matches the build file')
        return

    devel_view_name = get_devel_view_name(
        rosdistro_name, source_build_name, pull_request=False)
    pull_request_view_name = get_devel_view_name(
        rosdistro_name, source_build_name, pull_request=True)

    # all further configuration will be handled by either the Jenkins API
    # or by a generated groovy script
    from ros_buildfarm.jenkins import connect
    jenkins = connect(config.jenkins_url) if groovy_script is None else False

    view_configs = {}
    views = {}
    if build_file.test_commits_force is not False:
        views[devel_view_name] = configure_devel_view(
            jenkins, devel_view_name, dry_run=dry_run)
    if build_file.test_pull_requests_force is not False:
        views[pull_request_view_name] = configure_devel_view(
            jenkins, pull_request_view_name, dry_run=dry_run)
    if not jenkins:
        view_configs.update(views)
    groovy_data = {
        'dry_run': dry_run,
        'expected_num_views': len(view_configs),
    }

    repo_names = dist_file.repositories.keys()
    filtered_repo_names = build_file.filter_repositories(repo_names)

    devel_job_names = []
    pull_request_job_names = []
    job_configs = OrderedDict()
    for repo_name in sorted(repo_names):
        if whitelist_repository_names:
            if repo_name not in whitelist_repository_names:
                print(
                    "Skipping repository '%s' not in explicitly passed list" %
                    repo_name, file=sys.stderr)
                continue

        is_disabled = repo_name not in filtered_repo_names
        if is_disabled and build_file.skip_ignored_repositories:
            print("Skipping ignored repository '%s'" % repo_name,
                  file=sys.stderr)
            continue

        repo = dist_file.repositories[repo_name]
        if not repo.source_repository:
            print("Skipping repository '%s': no source section" % repo_name)
            continue
        if not repo.source_repository.version:
            print("Skipping repository '%s': no source version" % repo_name)
            continue

        job_types = []
        # check for testing commits
        if build_file.test_commits_force is False:
            print(("Skipping repository '%s': 'test_commits' is forced to " +
                   "false in the build file") % repo_name)
        elif repo.source_repository.test_commits is False:
            print(("Skipping repository '%s': 'test_commits' of the " +
                   "repository set to false") % repo_name)
        elif repo.source_repository.test_commits is None and \
                not build_file.test_commits_default:
            print(("Skipping repository '%s': 'test_commits' defaults to " +
                   "false in the build file") % repo_name)
        else:
            job_types.append('commit')

        if not is_disabled:
            # check for testing pull requests
            if build_file.test_pull_requests_force is False:
                # print(("Skipping repository '%s': 'test_pull_requests' " +
                #        "is forced to false in the build file") % repo_name)
                pass
            elif repo.source_repository.test_pull_requests is False:
                # print(("Skipping repository '%s': 'test_pull_requests' of " +
                #        "the repository set to false") % repo_name)
                pass
            elif repo.source_repository.test_pull_requests is None and \
                    not build_file.test_pull_requests_default:
                # print(("Skipping repository '%s': 'test_pull_requests' " +
                #        "defaults to false in the build file") % repo_name)
                pass
            else:
                print("Pull request job for repository '%s'" % repo_name)
                job_types.append('pull_request')

        for job_type in job_types:
            pull_request = job_type == 'pull_request'
            for os_name, os_code_name, arch in targets:
                try:
                    job_name, job_config = configure_devel_job(
                        config_url, rosdistro_name, source_build_name,
                        repo_name, os_name, os_code_name, arch, pull_request,
                        config=config, build_file=build_file,
                        index=index, dist_file=dist_file,
                        dist_cache=dist_cache, jenkins=jenkins, views=views,
                        is_disabled=is_disabled,
                        groovy_script=groovy_script,
                        dry_run=dry_run)
                    if not pull_request:
                        devel_job_names.append(job_name)
                    else:
                        pull_request_job_names.append(job_name)
                    if groovy_script is not None:
                        print("Configuration for job '%s'" % job_name)
                        job_configs[job_name] = job_config
                except JobValidationError as e:
                    print(e.message, file=sys.stderr)

    groovy_data['expected_num_jobs'] = len(job_configs)
    groovy_data['job_prefixes_and_names'] = {}

    devel_job_prefix = '%s__' % devel_view_name
    pull_request_job_prefix = '%s__' % pull_request_view_name
    if not whitelist_repository_names:
        groovy_data['job_prefixes_and_names']['devel'] = \
            (devel_job_prefix, devel_job_names)
        groovy_data['job_prefixes_and_names']['pull_request'] = \
            (pull_request_job_prefix, pull_request_job_names)

        if groovy_script is None:
            # delete obsolete jobs in these views
            from ros_buildfarm.jenkins import remove_jobs
            print('Removing obsolete devel jobs')
            remove_jobs(
                jenkins, devel_job_prefix, devel_job_names, dry_run=dry_run)
            print('Removing obsolete pull request jobs')
            remove_jobs(
                jenkins, pull_request_job_prefix, pull_request_job_names,
                dry_run=dry_run)
    if groovy_script is not None:
        print(
            "Writing groovy script '%s' to reconfigure %d views and %d jobs" %
            (groovy_script, len(view_configs), len(job_configs)))
        content = expand_template(
            'snippet/reconfigure_jobs.groovy.em', groovy_data)
        write_groovy_script_and_configs(
            groovy_script, content, job_configs, view_configs=view_configs)


Instruction:
Can you create a Python 3 function that
configures a single Jenkins devel job.

Response:
def configure_devel_job(
        config_url, rosdistro_name, source_build_name,
        repo_name, os_name, os_code_name, arch,
        pull_request=False,
        config=None, build_file=None,
        index=None, dist_file=None, dist_cache=None,
        jenkins=None, views=None,
        is_disabled=False,
        groovy_script=None,
        source_repository=None,
        build_targets=None,
        dry_run=False):
    """
    Configure a single Jenkins devel job.

    This includes the following steps:
    - clone the source repository to use
    - clone the ros_buildfarm repository
    - write the distribution repository keys into files
    - invoke the release/run_devel_job.py script
    """
    if config is None:
        config = get_config_index(config_url)
    if build_file is None:
        build_files = get_source_build_files(config, rosdistro_name)
        build_file = build_files[source_build_name]
    # Overwrite build_file.targets if build_targets is specified
    if build_targets is not None:
        build_file.targets = build_targets

    if index is None:
        index = get_index(config.rosdistro_index_url)
    if dist_file is None:
        dist_file = get_distribution_file(index, rosdistro_name, build_file)
        if not dist_file:
            raise JobValidationError(
                'No distribution file matches the build file')

    repo_names = dist_file.repositories.keys()

    if repo_name is not None:
        if repo_name not in repo_names:
            raise JobValidationError(
                "Invalid repository name '%s' " % repo_name +
                'choose one of the following: %s' %
                ', '.join(sorted(repo_names)))

        repo = dist_file.repositories[repo_name]
        if not repo.source_repository:
            raise JobValidationError(
                "Repository '%s' has no source section" % repo_name)
        if not repo.source_repository.version:
            raise JobValidationError(
                "Repository '%s' has no source version" % repo_name)
        source_repository = repo.source_repository

    if os_name not in build_file.targets.keys():
        raise JobValidationError(
            "Invalid OS name '%s' " % os_name +
            'choose one of the following: ' +
            ', '.join(sorted(build_file.targets.keys())))
    if os_code_name not in build_file.targets[os_name].keys():
        raise JobValidationError(
            "Invalid OS code name '%s' " % os_code_name +
            'choose one of the following: ' +
            ', '.join(sorted(build_file.targets[os_name].keys())))
    if arch not in build_file.targets[os_name][os_code_name]:
        raise JobValidationError(
            "Invalid architecture '%s' " % arch +
            'choose one of the following: %s' % ', '.join(sorted(
                build_file.targets[os_name][os_code_name])))

    if dist_cache is None and build_file.notify_maintainers:
        dist_cache = get_distribution_cache(index, rosdistro_name)
    if jenkins is None:
        from ros_buildfarm.jenkins import connect
        jenkins = connect(config.jenkins_url)
    if views is None:
        view_name = get_devel_view_name(
            rosdistro_name, source_build_name, pull_request=pull_request)
        configure_devel_view(jenkins, view_name, dry_run=dry_run)

    job_name = get_devel_job_name(
        rosdistro_name, source_build_name,
        repo_name, os_name, os_code_name, arch, pull_request)

    job_config = _get_devel_job_config(
        index, config, rosdistro_name, source_build_name,
        build_file, os_name, os_code_name, arch, source_repository,
        repo_name, pull_request, job_name, dist_cache=dist_cache,
        is_disabled=is_disabled)
    # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
    if isinstance(jenkins, object) and jenkins is not False:
        from ros_buildfarm.jenkins import configure_job
        configure_job(jenkins, job_name, job_config, dry_run=dry_run)

    return job_name, job_config


Instruction:
Explain what the following Python 3 code does
def configure_release_jobs(
        config_url, rosdistro_name, release_build_name, groovy_script=None,
        dry_run=False, whitelist_package_names=None):
    """
    Configure all Jenkins release jobs.

    L{configure_release_job} will be invoked for every released package and
    target which matches the build file criteria.

    Additionally a job to import Debian packages into the Debian repository is
    created.
    """
    config = get_config_index(config_url)
    build_files = get_release_build_files(config, rosdistro_name)
    build_file = build_files[release_build_name]

    index = get_index(config.rosdistro_index_url)

    # get targets
    platforms = []
    for os_name in build_file.targets.keys():
        for os_code_name in build_file.targets[os_name].keys():
            platforms.append((os_name, os_code_name))
    print('The build file contains the following targets:')
    for os_name, os_code_name in platforms:
        print('  - %s %s: %s' % (os_name, os_code_name, ', '.join(
            build_file.targets[os_name][os_code_name])))

    dist_file = get_distribution_file(index, rosdistro_name, build_file)
    if not dist_file:
        print('No distribution file matches the build file')
        return

    pkg_names = dist_file.release_packages.keys()
    filtered_pkg_names = build_file.filter_packages(pkg_names)
    explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names)
    if explicitly_ignored_pkg_names:
        print(('The following packages are being %s because of ' +
               'white-/blacklisting:') %
              ('ignored' if build_file.skip_ignored_packages else 'disabled'))
        for pkg_name in sorted(explicitly_ignored_pkg_names):
            print('  -', pkg_name)

    dist_cache = get_distribution_cache(index, rosdistro_name)

    if explicitly_ignored_pkg_names:
        # get direct dependencies from distro cache for each package
        direct_dependencies = {}
        for pkg_name in pkg_names:
            direct_dependencies[pkg_name] = _get_direct_dependencies(
                pkg_name, dist_cache, pkg_names) or set([])

        # find recursive downstream deps for all explicitly ignored packages
        ignored_pkg_names = set(explicitly_ignored_pkg_names)
        while True:
            implicitly_ignored_pkg_names = _get_downstream_package_names(
                ignored_pkg_names, direct_dependencies)
            if implicitly_ignored_pkg_names - ignored_pkg_names:
                ignored_pkg_names |= implicitly_ignored_pkg_names
                continue
            break
        implicitly_ignored_pkg_names = \
            ignored_pkg_names - explicitly_ignored_pkg_names

        if implicitly_ignored_pkg_names:
            print(('The following packages are being %s because their ' +
                   'dependencies are being ignored:') % ('ignored'
                  if build_file.skip_ignored_packages else 'disabled'))
            for pkg_name in sorted(implicitly_ignored_pkg_names):
                print('  -', pkg_name)
            filtered_pkg_names = \
                set(filtered_pkg_names) - implicitly_ignored_pkg_names

    # all further configuration will be handled by either the Jenkins API
    # or by a generated groovy script
    jenkins = False
    if groovy_script is None:
        from ros_buildfarm.jenkins import connect
        jenkins = connect(config.jenkins_url)

    all_view_configs = {}
    all_job_configs = OrderedDict()

    job_name, job_config = configure_import_package_job(
        config_url, rosdistro_name, release_build_name,
        config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run)
    if not jenkins:
        all_job_configs[job_name] = job_config

    job_name, job_config = configure_sync_packages_to_main_job(
        config_url, rosdistro_name, release_build_name,
        config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run)
    if not jenkins:
        all_job_configs[job_name] = job_config

    for os_name, os_code_name in platforms:
        for arch in sorted(build_file.targets[os_name][os_code_name]):
            job_name, job_config = configure_sync_packages_to_testing_job(
                config_url, rosdistro_name, release_build_name,
                os_code_name, arch,
                config=config, build_file=build_file, jenkins=jenkins,
                dry_run=dry_run)
            if not jenkins:
                all_job_configs[job_name] = job_config

    targets = []
    for os_name, os_code_name in platforms:
        targets.append((os_name, os_code_name, 'source'))
        for arch in build_file.targets[os_name][os_code_name]:
            targets.append((os_name, os_code_name, arch))
    views = configure_release_views(
        jenkins, rosdistro_name, release_build_name, targets,
        dry_run=dry_run)
    if not jenkins:
        all_view_configs.update(views)
    groovy_data = {
        'dry_run': dry_run,
        'expected_num_views': len(views),
    }

    # binary jobs must be generated in topological order
    from catkin_pkg.package import parse_package_string
    from ros_buildfarm.common import topological_order_packages
    pkgs = {}
    for pkg_name in pkg_names:
        if pkg_name not in dist_cache.release_package_xmls:
            print("Skipping package '%s': no released package.xml in cache" %
                  (pkg_name), file=sys.stderr)
            continue
        pkg_xml = dist_cache.release_package_xmls[pkg_name]
        pkg = parse_package_string(pkg_xml)
        pkgs[pkg_name] = pkg
    ordered_pkg_tuples = topological_order_packages(pkgs)

    other_build_files = [v for k, v in build_files.items() if k != release_build_name]

    all_source_job_names = []
    all_binary_job_names = []
    for pkg_name in [p.name for _, p in ordered_pkg_tuples]:
        if whitelist_package_names:
            if pkg_name not in whitelist_package_names:
                print("Skipping package '%s' not in the explicitly passed list" %
                      pkg_name, file=sys.stderr)
                continue

        pkg = dist_file.release_packages[pkg_name]
        repo_name = pkg.repository_name
        repo = dist_file.repositories[repo_name]
        is_disabled = pkg_name not in filtered_pkg_names
        if is_disabled and build_file.skip_ignored_packages:
            print("Skipping ignored package '%s' in repository '%s'" %
                  (pkg_name, repo_name), file=sys.stderr)
            continue
        if not repo.release_repository:
            print(("Skipping package '%s' in repository '%s': no release " +
                   "section") % (pkg_name, repo_name), file=sys.stderr)
            continue
        if not repo.release_repository.version:
            print(("Skipping package '%s' in repository '%s': no release " +
                   "version") % (pkg_name, repo_name), file=sys.stderr)
            continue

        for os_name, os_code_name in platforms:
            other_build_files_same_platform = []
            for other_build_file in other_build_files:
                if os_name not in other_build_file.targets:
                    continue
                if os_code_name not in other_build_file.targets[os_name]:
                    continue
                other_build_files_same_platform.append(other_build_file)

            try:
                source_job_names, binary_job_names, job_configs = \
                    configure_release_job(
                        config_url, rosdistro_name, release_build_name,
                        pkg_name, os_name, os_code_name,
                        config=config, build_file=build_file,
                        index=index, dist_file=dist_file,
                        dist_cache=dist_cache,
                        jenkins=jenkins, views=views,
                        generate_import_package_job=False,
                        generate_sync_packages_jobs=False,
                        is_disabled=is_disabled,
                        other_build_files_same_platform=other_build_files_same_platform,
                        groovy_script=groovy_script,
                        dry_run=dry_run)
                all_source_job_names += source_job_names
                all_binary_job_names += binary_job_names
                if groovy_script is not None:
                    print('Configuration for jobs: ' +
                          ', '.join(source_job_names + binary_job_names))
                    for source_job_name in source_job_names:
                        all_job_configs[source_job_name] = job_configs[source_job_name]
                    for binary_job_name in binary_job_names:
                        all_job_configs[binary_job_name] = job_configs[binary_job_name]
            except JobValidationError as e:
                print(e.message, file=sys.stderr)

    groovy_data['expected_num_jobs'] = len(all_job_configs)
    groovy_data['job_prefixes_and_names'] = {}

    # with an explicit list of packages we don't delete obsolete jobs
    if not whitelist_package_names:
        # delete obsolete binary jobs
        for os_name, os_code_name in platforms:
            for arch in build_file.targets[os_name][os_code_name]:
                binary_view = get_release_binary_view_name(
                    rosdistro_name, release_build_name,
                    os_name, os_code_name, arch)
                binary_job_prefix = '%s__' % binary_view

                excluded_job_names = set([
                    j for j in all_binary_job_names
                    if j.startswith(binary_job_prefix)])
                if groovy_script is None:
                    print("Removing obsolete binary jobs with prefix '%s'" %
                          binary_job_prefix)
                    from ros_buildfarm.jenkins import remove_jobs
                    remove_jobs(
                        jenkins, binary_job_prefix, excluded_job_names,
                        dry_run=dry_run)
                else:
                    binary_key = 'binary_%s_%s_%s' % \
                        (os_name, os_code_name, arch)
                    groovy_data['job_prefixes_and_names'][binary_key] = \
                        (binary_job_prefix, excluded_job_names)

        # delete obsolete source jobs
        # requires knowledge about all other release build files
        for os_name, os_code_name in platforms:
            other_source_job_names = []
            # get source job names for all other release build files
            for other_release_build_name in [
                    k for k in build_files.keys() if k != release_build_name]:
                other_build_file = build_files[other_release_build_name]
                other_dist_file = get_distribution_file(
                    index, rosdistro_name, other_build_file)
                if not other_dist_file:
                    continue

                if os_name not in other_build_file.targets or \
                        os_code_name not in other_build_file.targets[os_name]:
                    continue

                if other_build_file.skip_ignored_packages:
                    filtered_pkg_names = other_build_file.filter_packages(
                        pkg_names)
                else:
                    filtered_pkg_names = pkg_names
                for pkg_name in sorted(filtered_pkg_names):
                    pkg = other_dist_file.release_packages[pkg_name]
                    repo_name = pkg.repository_name
                    repo = other_dist_file.repositories[repo_name]
                    if not repo.release_repository:
                        continue
                    if not repo.release_repository.version:
                        continue

                    other_job_name = get_sourcedeb_job_name(
                        rosdistro_name, other_release_build_name,
                        pkg_name, os_name, os_code_name)
                    other_source_job_names.append(other_job_name)

            source_view_prefix = get_release_source_view_name(
                rosdistro_name, os_name, os_code_name)
            source_job_prefix = '%s__' % source_view_prefix
            excluded_job_names = set([
                j for j in (all_source_job_names + other_source_job_names)
                if j.startswith(source_job_prefix)])
            if groovy_script is None:
                print("Removing obsolete source jobs with prefix '%s'" %
                      source_job_prefix)
                from ros_buildfarm.jenkins import remove_jobs
                remove_jobs(
                    jenkins, source_job_prefix, excluded_job_names,
                    dry_run=dry_run)
            else:
                source_key = 'source_%s_%s' % (os_name, os_code_name)
                groovy_data['job_prefixes_and_names'][source_key] = (
                    source_job_prefix, excluded_job_names)

    if groovy_script is not None:
        print(
            "Writing groovy script '%s' to reconfigure %d views and %d jobs" %
            (groovy_script, len(all_view_configs), len(all_job_configs)))
        content = expand_template(
            'snippet/reconfigure_jobs.groovy.em', groovy_data)
        write_groovy_script_and_configs(
            groovy_script, content, all_job_configs,
            view_configs=all_view_configs)

Response: Configure all Jenkins release jobs.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def configure_release_job(
        config_url, rosdistro_name, release_build_name,
        pkg_name, os_name, os_code_name,
        config=None, build_file=None,
        index=None, dist_file=None, dist_cache=None,
        jenkins=None, views=None,
        generate_import_package_job=True,
        generate_sync_packages_jobs=True,
        is_disabled=False, other_build_files_same_platform=None,
        groovy_script=None,
        filter_arches=None,
        dry_run=False):
    """
    Configure a Jenkins release job.

    The following jobs are created for each package:
    - M source jobs, one for each OS node name
    - M * N binary jobs, one for each combination of OS code name and arch
    """
    if config is None:
        config = get_config_index(config_url)
    if build_file is None:
        build_files = get_release_build_files(config, rosdistro_name)
        build_file = build_files[release_build_name]

    if index is None:
        index = get_index(config.rosdistro_index_url)
    if dist_file is None:
        dist_file = get_distribution_file(index, rosdistro_name, build_file)
        if not dist_file:
            raise JobValidationError(
                'No distribution file matches the build file')

    pkg_names = dist_file.release_packages.keys()

    if pkg_name not in pkg_names:
        raise JobValidationError(
            "Invalid package name '%s' " % pkg_name +
            'choose one of the following: ' + ', '.join(sorted(pkg_names)))

    pkg = dist_file.release_packages[pkg_name]
    repo_name = pkg.repository_name
    repo = dist_file.repositories[repo_name]

    if not repo.release_repository:
        raise JobValidationError(
            "Repository '%s' has no release section" % repo_name)

    if not repo.release_repository.version:
        raise JobValidationError(
            "Repository '%s' has no release version" % repo_name)

    if os_name not in build_file.targets.keys():
        raise JobValidationError(
            "Invalid OS name '%s' " % os_name +
            'choose one of the following: ' +
            ', '.join(sorted(build_file.targets.keys())))

    if os_code_name not in build_file.targets[os_name].keys():
        raise JobValidationError(
            "Invalid OS code name '%s' " % os_code_name +
            'choose one of the following: ' +
            ', '.join(sorted(build_file.targets[os_name].keys())))

    if dist_cache is None and \
            (build_file.notify_maintainers or
             build_file.abi_incompatibility_assumed):
        dist_cache = get_distribution_cache(index, rosdistro_name)
    if jenkins is None:
        from ros_buildfarm.jenkins import connect
        jenkins = connect(config.jenkins_url)
    if views is None:
        targets = []
        targets.append((os_name, os_code_name, 'source'))
        for arch in build_file.targets[os_name][os_code_name]:
            targets.append((os_name, os_code_name, arch))
        configure_release_views(
            jenkins, rosdistro_name, release_build_name, targets,
            dry_run=dry_run)

    if generate_import_package_job:
        configure_import_package_job(
            config_url, rosdistro_name, release_build_name,
            config=config, build_file=build_file, jenkins=jenkins,
            dry_run=dry_run)

    if generate_sync_packages_jobs:
        configure_sync_packages_to_main_job(
            config_url, rosdistro_name, release_build_name,
            config=config, build_file=build_file, jenkins=jenkins,
            dry_run=dry_run)
        for arch in build_file.targets[os_name][os_code_name]:
            configure_sync_packages_to_testing_job(
                config_url, rosdistro_name, release_build_name,
                os_code_name, arch,
                config=config, build_file=build_file, jenkins=jenkins,
                dry_run=dry_run)

    source_job_names = []
    binary_job_names = []
    job_configs = {}

    # sourcedeb job
    # since sourcedeb jobs are potentially being shared across multiple build
    # files the configuration has to take all of them into account in order to
    # generate a job which all build files agree on
    source_job_name = get_sourcedeb_job_name(
        rosdistro_name, release_build_name,
        pkg_name, os_name, os_code_name)

    # while the package is disabled in the current build file
    # it might be used by sibling build files
    is_source_disabled = is_disabled
    if is_source_disabled and other_build_files_same_platform:
        # check if sourcedeb job is used by any other build file with the same platform
        for other_build_file in other_build_files_same_platform:
            if other_build_file.filter_packages([pkg_name]):
                is_source_disabled = False
                break

    job_config = _get_sourcedeb_job_config(
        config_url, rosdistro_name, release_build_name,
        config, build_file, os_name, os_code_name,
        pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache,
        is_disabled=is_source_disabled,
        other_build_files_same_platform=other_build_files_same_platform)
    # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
    if isinstance(jenkins, object) and jenkins is not False:
        from ros_buildfarm.jenkins import configure_job
        configure_job(jenkins, source_job_name, job_config, dry_run=dry_run)
    source_job_names.append(source_job_name)
    job_configs[source_job_name] = job_config

    dependency_names = []
    if build_file.abi_incompatibility_assumed:
        dependency_names = _get_direct_dependencies(
            pkg_name, dist_cache, pkg_names)
        # if dependencies are not yet available in rosdistro cache
        # skip binary jobs
        if dependency_names is None:
            print(("Skipping binary jobs for package '%s' because it is not " +
                   "yet in the rosdistro cache") % pkg_name, file=sys.stderr)
            return source_job_names, binary_job_names, job_configs

    # binarydeb jobs
    for arch in build_file.targets[os_name][os_code_name]:
        if filter_arches and arch not in filter_arches:
            continue

        job_name = get_binarydeb_job_name(
            rosdistro_name, release_build_name,
            pkg_name, os_name, os_code_name, arch)

        upstream_job_names = [source_job_name] + [
            get_binarydeb_job_name(
                rosdistro_name, release_build_name,
                dependency_name, os_name, os_code_name, arch)
            for dependency_name in dependency_names]

        job_config = _get_binarydeb_job_config(
            config_url, rosdistro_name, release_build_name,
            config, build_file, os_name, os_code_name, arch,
            pkg_name, repo_name, repo.release_repository,
            dist_cache=dist_cache, upstream_job_names=upstream_job_names,
            is_disabled=is_disabled)
        # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
        if isinstance(jenkins, object) and jenkins is not False:
            configure_job(jenkins, job_name, job_config, dry_run=dry_run)
        binary_job_names.append(job_name)
        job_configs[job_name] = job_config

    return source_job_names, binary_job_names, job_configs

Response: Configure a Jenkins release job.


Instruction:
Can you write a function in Python 3 where it
configures all Jenkins doc jobs for the given doc build.

Response:
def configure_doc_jobs(
        config_url, rosdistro_name, doc_build_name, groovy_script=None,
        dry_run=False, whitelist_repository_names=None):
    """
    Configure all Jenkins doc jobs.

    L{configure_doc_job} will be invoked for doc repository and target
    which matches the build file criteria.
    """
    config = get_config_index(config_url)
    build_files = get_doc_build_files(config, rosdistro_name)
    build_file = build_files[doc_build_name]

    index = get_index(config.rosdistro_index_url)

    dist_cache = None
    if build_file.notify_maintainers:
        dist_cache = get_distribution_cache(index, rosdistro_name)

    # get targets
    targets = []
    for os_name in build_file.targets.keys():
        for os_code_name in build_file.targets[os_name].keys():
            for arch in build_file.targets[os_name][os_code_name]:
                targets.append((os_name, os_code_name, arch))
    print('The build file contains the following targets:')
    for os_name, os_code_name, arch in targets:
        print('  -', os_name, os_code_name, arch)

    dist_file = get_distribution_file(index, rosdistro_name, build_file)
    if not dist_file:
        print('No distribution file matches the build file')
        return

    doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name)

    # all further configuration will be handled by either the Jenkins API
    # or by a generated groovy script
    from ros_buildfarm.jenkins import connect
    jenkins = connect(config.jenkins_url) if groovy_script is None else False

    view_configs = {}
    views = {}
    views[doc_view_name] = configure_doc_view(
        jenkins, doc_view_name, dry_run=dry_run)
    if not jenkins:
        view_configs.update(views)
    groovy_data = {
        'dry_run': dry_run,
        'expected_num_views': len(view_configs),
    }

    repo_names = dist_file.repositories.keys()
    filtered_repo_names = build_file.filter_repositories(repo_names)

    job_names = []
    job_configs = OrderedDict()
    for repo_name in sorted(repo_names):
        if whitelist_repository_names:
            if repo_name not in whitelist_repository_names:
                print(
                    "Skipping repository '%s' not in explicitly passed list" %
                    repo_name, file=sys.stderr)
                continue
        is_disabled = repo_name not in filtered_repo_names
        if is_disabled and build_file.skip_ignored_repositories:
            print("Skipping ignored repository '%s'" % repo_name,
                  file=sys.stderr)
            continue

        repo = dist_file.repositories[repo_name]
        if not repo.doc_repository:
            print("Skipping repository '%s': no doc section" % repo_name)
            continue
        if not repo.doc_repository.version:
            print("Skipping repository '%s': no doc version" % repo_name)
            continue

        for os_name, os_code_name, arch in targets:
            try:
                job_name, job_config = configure_doc_job(
                    config_url, rosdistro_name, doc_build_name,
                    repo_name, os_name, os_code_name, arch,
                    config=config, build_file=build_file,
                    index=index, dist_file=dist_file,
                    dist_cache=dist_cache, jenkins=jenkins, views=views,
                    is_disabled=is_disabled,
                    groovy_script=groovy_script,
                    dry_run=dry_run)
                job_names.append(job_name)
                if groovy_script is not None:
                    print("Configuration for job '%s'" % job_name)
                    job_configs[job_name] = job_config
            except JobValidationError as e:
                print(e.message, file=sys.stderr)

    groovy_data['expected_num_jobs'] = len(job_configs)
    groovy_data['job_prefixes_and_names'] = {}

    job_prefix = '%s__' % doc_view_name
    if not whitelist_repository_names:
        groovy_data['job_prefixes_and_names']['doc'] = (job_prefix, job_names)

        if groovy_script is None:
            # delete obsolete jobs in this view
            from ros_buildfarm.jenkins import remove_jobs
            print('Removing obsolete doc jobs')
            remove_jobs(jenkins, job_prefix, job_names, dry_run=dry_run)
    if groovy_script is not None:
        print(
            "Writing groovy script '%s' to reconfigure %d views and %d jobs" %
            (groovy_script, len(view_configs), len(job_configs)))
        content = expand_template(
            'snippet/reconfigure_jobs.groovy.em', groovy_data)
        write_groovy_script_and_configs(
            groovy_script, content, job_configs, view_configs=view_configs)


Instruction:
Make a summary of the following Python 3 code
def configure_doc_job(
        config_url, rosdistro_name, doc_build_name,
        repo_name, os_name, os_code_name, arch,
        config=None, build_file=None,
        index=None, dist_file=None, dist_cache=None,
        jenkins=None, views=None,
        is_disabled=False,
        groovy_script=None,
        doc_repository=None,
        dry_run=False):
    """
    Configure a single Jenkins doc job.

    This includes the following steps:
    - clone the doc repository to use
    - clone the ros_buildfarm repository
    - write the distribution repository keys into files
    - invoke the run_doc_job.py script
    """
    if config is None:
        config = get_config_index(config_url)
    if build_file is None:
        build_files = get_doc_build_files(config, rosdistro_name)
        build_file = build_files[doc_build_name]

    if index is None:
        index = get_index(config.rosdistro_index_url)
    if dist_file is None:
        dist_file = get_distribution_file(index, rosdistro_name, build_file)
        if not dist_file:
            raise JobValidationError(
                'No distribution file matches the build file')

    repo_names = dist_file.repositories.keys()

    if repo_name is not None:
        if repo_name not in repo_names:
            raise JobValidationError(
                "Invalid repository name '%s' " % repo_name +
                'choose one of the following: %s' %
                ', '.join(sorted(repo_names)))

        repo = dist_file.repositories[repo_name]
        if not repo.doc_repository:
            raise JobValidationError(
                "Repository '%s' has no doc section" % repo_name)
        if not repo.doc_repository.version:
            raise JobValidationError(
                "Repository '%s' has no doc version" % repo_name)
        doc_repository = repo.doc_repository

    if os_name not in build_file.targets.keys():
        raise JobValidationError(
            "Invalid OS name '%s' " % os_name +
            'choose one of the following: ' +
            ', '.join(sorted(build_file.targets.keys())))
    if os_code_name not in build_file.targets[os_name].keys():
        raise JobValidationError(
            "Invalid OS code name '%s' " % os_code_name +
            'choose one of the following: ' +
            ', '.join(sorted(build_file.targets[os_name].keys())))
    if arch not in build_file.targets[os_name][os_code_name]:
        raise JobValidationError(
            "Invalid architecture '%s' " % arch +
            'choose one of the following: %s' % ', '.join(sorted(
                build_file.targets[os_name][os_code_name])))

    if dist_cache is None and build_file.notify_maintainers:
        dist_cache = get_distribution_cache(index, rosdistro_name)
    if jenkins is None:
        from ros_buildfarm.jenkins import connect
        jenkins = connect(config.jenkins_url)
    if views is None:
        view_name = get_doc_view_name(
            rosdistro_name, doc_build_name)
        configure_doc_view(jenkins, view_name, dry_run=dry_run)

    job_name = get_doc_job_name(
        rosdistro_name, doc_build_name,
        repo_name, os_name, os_code_name, arch)

    job_config = _get_doc_job_config(
        config, config_url, rosdistro_name, doc_build_name,
        build_file, os_name, os_code_name, arch, doc_repository,
        repo_name, dist_cache=dist_cache, is_disabled=is_disabled)
    # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
    if isinstance(jenkins, object) and jenkins is not False:
        from ros_buildfarm.jenkins import configure_job
        configure_job(jenkins, job_name, job_config, dry_run=dry_run)

    return job_name, job_config

Response: Configure a single Jenkins doc job.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_affected_by_sync(
        package_descriptors, targets,
        testing_repo_data, main_repo_data):
    """
    For each package and target check if it is affected by a sync.

    This is the case when the package version in the testing repo is different
    from the version in the main repo.

    :return: a dict indexed by package names containing
      dicts indexed by targets containing a boolean flag
    """
    affected_by_sync = {}
    for package_descriptor in package_descriptors.values():
        pkg_name = package_descriptor.pkg_name
        debian_pkg_name = package_descriptor.debian_pkg_name

        affected_by_sync[pkg_name] = {}
        for target in targets:
            testing_version = _strip_version_suffix(
                testing_repo_data.get(target, {}).get(debian_pkg_name, None))
            main_version = _strip_version_suffix(
                main_repo_data.get(target, {}).get(debian_pkg_name, None))

            affected_by_sync[pkg_name][target] = \
                testing_version != main_version
    return affected_by_sync

Response: Returns a dict indexed by package names containing a boolean flag indicating if the version of the package in testing repo is different from the version in main repo.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_regressions(
        package_descriptors, targets,
        building_repo_data, testing_repo_data, main_repo_data):
    """
    For each package and target check if it is a regression.

    This is the case if the main repo contains a package version which is
    higher then in any of the other repos or if any of the other repos does not
    contain that package at all.

    :return: a dict indexed by package names containing
      dicts indexed by targets containing a boolean flag
    """
    regressions = {}
    for package_descriptor in package_descriptors.values():
        pkg_name = package_descriptor.pkg_name
        debian_pkg_name = package_descriptor.debian_pkg_name

        regressions[pkg_name] = {}
        for target in targets:
            regressions[pkg_name][target] = False
            main_version = \
                main_repo_data.get(target, {}).get(debian_pkg_name, None)
            if main_version is not None:
                main_ver_loose = LooseVersion(main_version)
                for repo_data in [building_repo_data, testing_repo_data]:
                    version = \
                        repo_data.get(target, {}).get(debian_pkg_name, None)
                    if not version or main_ver_loose > LooseVersion(version):
                        regressions[pkg_name][target] = True
    return regressions

Response: Returns a dictionary indexed by package names containing a boolean flag indicating if the main repo contains a version that is higher than the main version of the package.


Instruction:
Make a summary of the following Python 3 code
def get_version_status(
        package_descriptors, targets, repos_data,
        strip_version=False, strip_os_code_name=False):
    """
    For each package and target check if it is affected by a sync.

    This is the case when the package version in the testing repo is different
    from the version in the main repo.

    :return: a dict indexed by package names containing
      dicts indexed by targets containing
      a list of status strings (one for each repo)
    """
    status = {}
    for package_descriptor in package_descriptors.values():
        pkg_name = package_descriptor.pkg_name
        debian_pkg_name = package_descriptor.debian_pkg_name
        ref_version = package_descriptor.version
        if strip_version:
            ref_version = _strip_version_suffix(ref_version)

        status[pkg_name] = {}
        for target in targets:
            statuses = []
            for repo_data in repos_data:
                version = repo_data.get(target, {}).get(debian_pkg_name, None)
                if strip_version:
                    version = _strip_version_suffix(version)
                if strip_os_code_name:
                    version = _strip_os_code_name_suffix(
                        version, target.os_code_name)

                if ref_version:
                    if not version:
                        statuses.append('missing')
                    elif version.startswith(ref_version):  # including equal
                        statuses.append('equal')
                    else:
                        if _version_is_gt_other(version, ref_version):
                            statuses.append('higher')
                        else:
                            statuses.append('lower')
                else:
                    if not version:
                        statuses.append('ignore')
                    else:
                        statuses.append('obsolete')
            status[pkg_name][target] = statuses
    return status

Response: Get the status of a set of packages and targets.


Instruction:
Implement a Python 3 function for
stripping trailing junk from the version number.

Response:
def _strip_version_suffix(version):
    """
    Remove trailing junk from the version number.

    >>> strip_version_suffix('')
    ''
    >>> strip_version_suffix('None')
    'None'
    >>> strip_version_suffix('1.2.3-4trusty-20140131-1359-+0000')
    '1.2.3-4'
    >>> strip_version_suffix('1.2.3-foo')
    '1.2.3'
    """
    global version_regex
    if not version:
        return version
    match = version_regex.search(version)
    return match.group(0) if match else version


Instruction:
Implement a Python 3 function for
returning a dict indexed by package names containing a boolean flag indicating if the version in one repo is equal for all targets.

Response:
def get_homogeneous(package_descriptors, targets, repos_data):
    """
    For each package check if the version in one repo is equal for all targets.

    The version could be different in different repos though.

    :return: a dict indexed by package names containing a boolean flag
    """
    homogeneous = {}
    for package_descriptor in package_descriptors.values():
        pkg_name = package_descriptor.pkg_name
        debian_pkg_name = package_descriptor.debian_pkg_name

        versions = []
        for repo_data in repos_data:
            versions.append(set([]))
            for target in targets:
                version = _strip_version_suffix(
                    repo_data.get(target, {}).get(debian_pkg_name, None))
                versions[-1].add(version)
        homogeneous[pkg_name] = max([len(v) for v in versions]) == 1
    return homogeneous


Instruction:
Write a Python 3 script to
get the number of packages per target and repository.

Response:
def get_package_counts(package_descriptors, targets, repos_data):
    """
    Get the number of packages per target and repository.

    :return: a dict indexed by targets containing
      a list of integer values (one for each repo)
    """
    counts = {}
    for target in targets:
        counts[target] = [0] * len(repos_data)
    for package_descriptor in package_descriptors.values():
        debian_pkg_name = package_descriptor.debian_pkg_name

        for target in targets:
            for i, repo_data in enumerate(repos_data):
                version = repo_data.get(target, {}).get(debian_pkg_name, None)
                if version:
                    counts[target][i] += 1
    return counts