Instruction:
Implement a function in Python 3 to
convert RGB to ANSI 256 color

Response:
def rgb_to_ansi256(r, g, b):
    """
    Convert RGB to ANSI 256 color
    """
    if r == g and g == b:
        if r < 8:
            return 16
        if r > 248:
            return 231

        return round(((r - 8) / 247.0) * 24) + 232

    ansi_r = 36 * round(r / 255.0 * 5.0)
    ansi_g = 6 * round(g / 255.0 * 5.0)
    ansi_b = round(b / 255.0 * 5.0)
    ansi = 16 + ansi_r + ansi_g + ansi_b
    return ansi


Instruction:
Write a Python 3 function for
converting RGB to ANSI 16 color

Response:
def rgb_to_ansi16(r, g, b, use_bright=False):
    """
    Convert RGB to ANSI 16 color
    """
    ansi_b = round(b / 255.0) << 2
    ansi_g = round(g / 255.0) << 1
    ansi_r = round(r / 255.0)
    ansi = (90 if use_bright else 30) + (ansi_b | ansi_g | ansi_r)

    return ansi


Instruction:
Can you implement a function in Python 3 that
converts the given hex string to a valid RGB channel triplet.

Response:
def hex_to_rgb(value):
    """
    Convert the given hex string to a
    valid RGB channel triplet.
    """
    value = value.lstrip('#')
    check_hex(value)

    length = len(value)
    step = int(length / 3)
    return tuple(int(value[i:i+step], 16) for i in range(0, length, step))


Instruction:
Here you have a function in Python 3, explain what it does
def check_hex(value):
    length = len(value)
    if length not in (3, 6):
        raise ValueError('Hex string #{} is too long'.format(value))

    regex = r'[0-9a-f]{{{length}}}'.format(length=length)
    if not re.search(regex, value, re.I):
        raise ValueError('Invalid Hex String: #{}'.format(value))

Response: Check if the given hex value is a valid RGB color


Instruction:
How would you explain what the following Python 3 function does
def show():
    with colorful.with_style('solarized') as c:
        # modifiers
        sys.stdout.write(c.bold('bold') + ' ')
        sys.stdout.write(c.dimmed('dimmed') + ' ')
        sys.stdout.write(c.italic('italic') + ' ')
        sys.stdout.write(c.underlined('underlined') + ' ')
        sys.stdout.write(c.inversed('inversed') + ' ')
        sys.stdout.write(c.concealed('concealed') + ' ')
        sys.stdout.write(c.struckthrough('struckthrough') + '\n')

        # foreground colors
        sys.stdout.write(c.yellow('yellow') + ' ')
        sys.stdout.write(c.red('orange') + ' ')
        sys.stdout.write(c.red('red') + ' ')
        sys.stdout.write(c.magenta('magenta') + ' ')
        sys.stdout.write(c.magenta('violet') + ' ')
        sys.stdout.write(c.blue('blue') + ' ')
        sys.stdout.write(c.cyan('cyan') + ' ')
        sys.stdout.write(c.green('green') + '\n')

        # background colors
        sys.stdout.write(c.on_yellow('yellow') + ' ')
        sys.stdout.write(c.on_red('orange') + ' ')
        sys.stdout.write(c.on_red('red') + ' ')
        sys.stdout.write(c.on_magenta('magenta') + ' ')
        sys.stdout.write(c.on_magenta('violet') + ' ')
        sys.stdout.write(c.on_blue('blue') + ' ')
        sys.stdout.write(c.on_cyan('cyan') + ' ')
        sys.stdout.write(c.on_green('green') + '\n')

Response: Show the modifiers and colors of the current language.


Instruction:
Make a summary of the following Python 3 code
def translate_rgb_to_ansi_code(red, green, blue, offset, colormode):
    if colormode == terminal.NO_COLORS:  # colors are disabled, thus return empty string
        return '', ''

    if colormode == terminal.ANSI_8_COLORS or colormode == terminal.ANSI_16_COLORS:
        color_code = ansi.rgb_to_ansi16(red, green, blue)
        start_code = ansi.ANSI_ESCAPE_CODE.format(
            code=color_code + offset - ansi.FOREGROUND_COLOR_OFFSET)
        end_code = ansi.ANSI_ESCAPE_CODE.format(code=offset + ansi.COLOR_CLOSE_OFFSET)
        return start_code, end_code

    if colormode == terminal.ANSI_256_COLORS:
        color_code = ansi.rgb_to_ansi256(red, green, blue)
        start_code = ansi.ANSI_ESCAPE_CODE.format(code='{base};5;{code}'.format(
            base=8 + offset, code=color_code))
        end_code = ansi.ANSI_ESCAPE_CODE.format(code=offset + ansi.COLOR_CLOSE_OFFSET)
        return start_code, end_code

    if colormode == terminal.TRUE_COLORS:
        start_code = ansi.ANSI_ESCAPE_CODE.format(code='{base};2;{red};{green};{blue}'.format(
            base=8 + offset, red=red, green=green, blue=blue))
        end_code = ansi.ANSI_ESCAPE_CODE.format(code=offset + ansi.COLOR_CLOSE_OFFSET)
        return start_code, end_code

    raise ColorfulError('invalid color mode "{0}"'.format(colormode))

Response: Translate the given RGB color into the appropriate ANSI escape code.


Instruction:
Implement a Python 3 function for
translating a given color name to an ANSI escape code.

Response:
def translate_colorname_to_ansi_code(colorname, offset, colormode, colorpalette):
    """
    Translate the given color name to a valid
    ANSI escape code.

    :parma str colorname: the name of the color to resolve
    :parma str offset: the offset for the color code
    :param int colormode: the color mode to use. See ``translate_rgb_to_ansi_code``
    :parma dict colorpalette: the color palette to use for the color name mapping

    :returns str: the color as ANSI escape code

    :raises ColorfulError: if the given color name is invalid
    """
    try:
        red, green, blue = colorpalette[colorname]
    except KeyError:
        raise ColorfulError('the color "{0}" is unknown. Use a color in your color palette (by default: X11 rgb.txt)'.format(  # noqa
            colorname))
    else:
        return translate_rgb_to_ansi_code(red, green, blue, offset, colormode)


Instruction:
Make a summary of the following Python 3 code
def resolve_modifier_to_ansi_code(modifiername, colormode):
    if colormode == terminal.NO_COLORS:  # return empty string if colors are disabled
        return '', ''

    try:
        start_code, end_code = ansi.MODIFIERS[modifiername]
    except KeyError:
        raise ColorfulError('the modifier "{0}" is unknown. Use one of: {1}'.format(
            modifiername, ansi.MODIFIERS.keys()))
    else:
        return ansi.ANSI_ESCAPE_CODE.format(
            code=start_code), ansi.ANSI_ESCAPE_CODE.format(
                code=end_code)

Response: Resolves the given modifier name to a valid ANSI escape code.


Instruction:
Given the following Python 3 function, write the documentation
def translate_style(style, colormode, colorpalette):
    style_parts = iter(style.split('_'))

    ansi_start_sequence = []
    ansi_end_sequence = []

    try:
        # consume all modifiers
        part = None
        for mod_part in style_parts:
            part = mod_part
            if part not in ansi.MODIFIERS:
                break  # all modifiers have been consumed

            mod_start_code, mod_end_code = resolve_modifier_to_ansi_code(part, colormode)
            ansi_start_sequence.append(mod_start_code)
            ansi_end_sequence.append(mod_end_code)
        else:  # we've consumed all parts, thus we can exit
            raise StopIteration()

        # next part has to be a foreground color or the 'on' keyword
        # which means we have to consume background colors
        if part != 'on':
            ansi_start_code, ansi_end_code = translate_colorname_to_ansi_code(
                part, ansi.FOREGROUND_COLOR_OFFSET, colormode, colorpalette)
            ansi_start_sequence.append(ansi_start_code)
            ansi_end_sequence.append(ansi_end_code)
            # consume the required 'on' keyword after the foreground color
            next(style_parts)

        # next part has to be the background color
        part = next(style_parts)
        ansi_start_code, ansi_end_code = translate_colorname_to_ansi_code(
            part, ansi.BACKGROUND_COLOR_OFFSET, colormode, colorpalette)
        ansi_start_sequence.append(ansi_start_code)
        ansi_end_sequence.append(ansi_end_code)
    except StopIteration:  # we've consumed all parts of the styling string
        pass

    # construct and return ANSI escape code sequence
    return ''.join(ansi_start_sequence), ''.join(ansi_end_sequence)

Response: Translate the given style to an ANSI escape code.


Instruction:
Create a Python 3 function for
returning a string that is a style of the given string according to the given ANSI style string.

Response:
def style_string(string, ansi_style, colormode, nested=False):
    """
    Style the given string according to the given
    ANSI style string.

    :param str string: the string to style
    :param tuple ansi_style: the styling string returned by ``translate_style``
    :param int colormode: the color mode to use. See ``translate_rgb_to_ansi_code``

    :returns: a string containing proper ANSI sequence
    """
    ansi_start_code, ansi_end_code = ansi_style

    # replace nest placeholders with the current begin style
    if PY2:
        if isinstance(string, str):
            string = string.decode(DEFAULT_ENCODING)
    string = UNICODE(string).replace(ansi.NEST_PLACEHOLDER, ansi_start_code)

    return '{start_code}{string}{end_code}{nest_ph}'.format(
            start_code=ansi_start_code,
            string=string,
            end_code=ansi_end_code,
            nest_ph=ansi.NEST_PLACEHOLDER if nested else '')


Instruction:
Given the following Python 3 function, write the documentation
def colorpalette(self, colorpalette):
        if isinstance(colorpalette, str):  # we assume it's a path to a color file
            colorpalette = colors.parse_colors(colorpalette)

        self._colorpalette = colors.sanitize_color_palette(colorpalette)

Response: Set the colorpalette which should be used by the log file.


Instruction:
Here you have a function in Python 3, explain what it does
def setup(self, colormode=None, colorpalette=None, extend_colors=False):
        if colormode:
            self.colormode = colormode

        if colorpalette:
            if extend_colors:
                self.update_palette(colorpalette)
            else:
                self.colorpalette = colorpalette

Response: Setup this colorful object by setting a colormode and colorpalette.


Instruction:
Can you create a Python 3 function that
uses a predefined style as color palette

Response:
def use_style(self, style_name):
        """
        Use a predefined style as color palette

        :param str style_name: the name of the style
        """
        try:
            style = getattr(styles, style_name.upper())
        except AttributeError:
            raise ColorfulError('the style "{0}" is undefined'.format(
                style_name))
        else:
            self.colorpalette = style


Instruction:
Can you generate the documentation for the following Python 3 function
def format(self, string, *args, **kwargs):
        return string.format(c=self, *args, **kwargs)

Response: Format the given string with the given args and kwargs.


Instruction:
Here you have a function in Python 3, explain what it does
def print(self, *objects, **options):
        # NOTE: change signature to same as print() built-in function as
        #       soon as Python 2.7 is not supported anymore.
        #       If causes problems because of the keyword args after *args
        allowed_options = {'sep', 'end', 'file', 'flush'}
        given_options = set(options.keys())
        if not given_options.issubset(allowed_options):
            raise TypeError('Colorful.print() got unexpected keyword arguments: {0}'.format(
                ', '.join(given_options.difference(allowed_options))))

        sep = options.get('sep', ' ')
        end = options.get('end', '\n')
        file = options.get('file', sys.stdout)
        flush = options.get('flush', False)

        styled_objects = [self.format(o) for o in objects]
        print(*styled_objects, sep=sep, end=end, file=file)

        # NOTE: if Python 2.7 support is dropped we can directly forward the
        #       flush keyword argument to the print() function.
        if flush:
            file.flush()

Response: Print the given objects to the given file stream.


Instruction:
Write a Python 3 function that can
read attribute from sysfs and return as string

Response:
def readattr(path, name):
    """
    Read attribute from sysfs and return as string
    """
    try:
        f = open(USB_SYS_PREFIX + path + "/" + name)
        return f.readline().rstrip("\n")
    except IOError:
        return None


Instruction:
Given the following Python 3 function, write the documentation
def find_ports(device):
    bus_id = device.bus
    dev_id = device.address
    for dirent in os.listdir(USB_SYS_PREFIX):
        matches = re.match(USB_PORTS_STR + '$', dirent)
        if matches:
            bus_str = readattr(dirent, 'busnum')
            if bus_str:
                busnum = float(bus_str)
            else:
                busnum = None
            dev_str = readattr(dirent, 'devnum')
            if dev_str:
                devnum = float(dev_str)
            else:
                devnum = None
            if busnum == bus_id and devnum == dev_id:
                return str(matches.groups()[1])

Response: Find the port chain a device is plugged on.


Instruction:
How would you implement a function in Python 3 that
sets device calibration data based on settings in the temper. conf file.

Response:
def set_calibration_data(self, scale=None, offset=None):
        """
        Set device calibration data based on settings in /etc/temper.conf.
        """
        if scale is not None and offset is not None:
            self._scale = scale
            self._offset = offset
        elif scale is None and offset is None:
            self._scale = 1.0
            self._offset = 0.0
            try:
                f = open('/etc/temper.conf', 'r')
            except IOError:
                f = None
            if f:
                lines = f.read().split('\n')
                f.close()
                for line in lines:
                    matches = re.match(CALIB_LINE_STR, line)
                    if matches:
                        bus = int(matches.groups()[0])
                        ports = matches.groups()[1]
                        scale = float(matches.groups()[2])
                        offset = float(matches.groups()[3])
                        if (str(ports) == str(self._ports)) and (str(bus) == str(self._bus)):
                            self._scale = scale
                            self._offset = offset
        else:
            raise RuntimeError("Must set both scale and offset, or neither")


Instruction:
Write a Python 3 function for
getting data from the USB device.

Response:
def get_data(self, reset_device=False):
        """
        Get data from the USB device.
        """
        try:
            if reset_device:
                self._device.reset()

            # detach kernel driver from both interfaces if attached, so we can set_configuration()
            for interface in [0,1]:
                if self._device.is_kernel_driver_active(interface):
                    LOGGER.debug('Detaching kernel driver for interface %d '
                        'of %r on ports %r', interface, self._device, self._ports)
                    self._device.detach_kernel_driver(interface)

            self._device.set_configuration()

            # Prevent kernel message:
            # "usbfs: process <PID> (python) did not claim interface x before use"
            # This will become unnecessary once pull-request #124 for
            # PyUSB has been accepted and we depend on a fixed release
            # of PyUSB.  Until then, and even with the fix applied, it
            # does not hurt to explicitly claim the interface.
            usb.util.claim_interface(self._device, INTERFACE)

                # Turns out we don't actually need that ctrl_transfer.
                # Disabling this reduces number of USBErrors from ~7/30 to 0!
                #self._device.ctrl_transfer(bmRequestType=0x21, bRequest=0x09,
                #    wValue=0x0201, wIndex=0x00, data_or_wLength='\x01\x01',
                #    timeout=TIMEOUT)

            # Magic: Our TEMPerV1.4 likes to be asked twice.  When
            # only asked once, it get's stuck on the next access and
            # requires a reset.
            self._control_transfer(COMMANDS['temp'])
            self._interrupt_read()

            # Turns out a whole lot of that magic seems unnecessary.
            #self._control_transfer(COMMANDS['ini1'])
            #self._interrupt_read()
            #self._control_transfer(COMMANDS['ini2'])
            #self._interrupt_read()
            #self._interrupt_read()

            # Get temperature
            self._control_transfer(COMMANDS['temp'])
            temp_data = self._interrupt_read()

            # Get humidity
            if self._device.product == 'TEMPer1F_H1_V1.4':
                humidity_data = temp_data
            else:
                humidity_data = None

            # Combine temperature and humidity data
            data = {'temp_data': temp_data, 'humidity_data': humidity_data}

            # Be a nice citizen and undo potential interface claiming.
            # Also see: https://github.com/walac/pyusb/blob/master/docs/tutorial.rst#dont-be-selfish
            usb.util.dispose_resources(self._device)
            return data
        except usb.USBError as err:
            if not reset_device:
                LOGGER.warning("Encountered %s, resetting %r and trying again.", err, self._device)
                return self.get_data(True)

            # Catch the permissions exception and add our message
            if "not permitted" in str(err):
                raise Exception(
                    "Permission problem accessing USB. "
                    "Maybe I need to run as root?")
            else:
                LOGGER.error(err)
                raise


Instruction:
Can you generate the documentation for the following Python 3 function
def get_temperature(self, format='celsius', sensor=0):
        results = self.get_temperatures(sensors=[sensor,])

        if format == 'celsius':
            return results[sensor]['temperature_c']
        elif format == 'fahrenheit':
            return results[sensor]['temperature_f']
        elif format == 'millicelsius':
            return results[sensor]['temperature_mc']
        else:
            raise ValueError("Unknown format")

Response: Get the temperature in the specified format.


Instruction:
Can you implement a function in Python 3 that
gets the temperature of a specific sensor.

Response:
def get_temperatures(self, sensors=None):
        """
        Get device temperature reading.

        Params:
        - sensors: optional list of sensors to get a reading for, examples:
          [0,] - get reading for sensor 0
          [0, 1,] - get reading for sensors 0 and 1
          None - get readings for all sensors
        """
        _sensors = sensors
        if _sensors is None:
            _sensors = list(range(0, self._sensor_count))

        if not set(_sensors).issubset(list(range(0, self._sensor_count))):
            raise ValueError(
                'Some or all of the sensors in the list %s are out of range '
                'given a sensor_count of %d.  Valid range: %s' % (
                    _sensors,
                    self._sensor_count,
                    list(range(0, self._sensor_count)),
                )
            )

        data = self.get_data()
        data = data['temp_data']

        results = {}

        # Interpret device response
        for sensor in _sensors:
            offset = self.lookup_offset(sensor)
            celsius = struct.unpack_from('>h', data, offset)[0] / 256.0
            # Apply scaling and offset (if any)
            celsius = celsius * self._scale + self._offset
            results[sensor] = {
                'ports': self.get_ports(),
                'bus': self.get_bus(),
                'sensor': sensor,
                'temperature_f': celsius * 1.8 + 32.0,
                'temperature_c': celsius,
                'temperature_mc': celsius * 1000,
                'temperature_k': celsius + 273.15,
            }

        return results


Instruction:
Implement a Python 3 function for
getting the device humidity reading.

Response:
def get_humidity(self, sensors=None):
        """
        Get device humidity reading.

        Params:
        - sensors: optional list of sensors to get a reading for, examples:
          [0,] - get reading for sensor 0
          [0, 1,] - get reading for sensors 0 and 1
          None - get readings for all sensors
        """
        _sensors = sensors
        if _sensors is None:
            _sensors = list(range(0, self._sensor_count))

        if not set(_sensors).issubset(list(range(0, self._sensor_count))):
            raise ValueError(
                'Some or all of the sensors in the list %s are out of range '
                'given a sensor_count of %d.  Valid range: %s' % (
                    _sensors,
                    self._sensor_count,
                    list(range(0, self._sensor_count)),
                )
            )

        data = self.get_data()
        data = data['humidity_data']

        results = {}

        # Interpret device response
        for sensor in _sensors:
            offset = self.lookup_humidity_offset(sensor)
            if offset is None:
                continue
            humidity = (struct.unpack_from('>H', data, offset)[0] * 32) / 1000.0
            results[sensor] = {
                'ports': self.get_ports(),
                'bus': self.get_bus(),
                'sensor': sensor,
                'humidity_pc': humidity,
            }

        return results


Instruction:
Make a summary of the following Python 3 code
def _control_transfer(self, data):
        LOGGER.debug('Ctrl transfer: %r', data)
        self._device.ctrl_transfer(bmRequestType=0x21, bRequest=0x09,
            wValue=0x0200, wIndex=0x01, data_or_wLength=data, timeout=TIMEOUT)

Response: Send a control transfer request to the device.


Instruction:
How would you explain what the following Python 3 function does
def _interrupt_read(self):
        data = self._device.read(ENDPOINT, REQ_INT_LEN, timeout=TIMEOUT)
        LOGGER.debug('Read data: %r', data)
        return data

Response: Read data from device.


Instruction:
Can you tell what is the following Python 3 function doing
def __check_looks_like_uri(self, uri):
        if uri.split('/')[2] == 'raw.githubusercontent.com':
            return True
        elif uri.split('/')[2] == 'github.com':
            if uri.split('/')[5] == 'raw':
                return True
        else:
            raise GithubFileNotFound('URI %s is not a valid link to a raw file in Github' % uri)

Response: Checks if the URI looks like a RAW uri in Github


Instruction:
Write a Python 3 script to
read the file from Github and returns the content

Response:
def read_file_from_uri(self, uri):
        """Reads the file from Github

        :param uri: URI of the Github raw File

        :returns: UTF-8 text with the content
        """
        logger.debug("Reading %s" % (uri))

        self.__check_looks_like_uri(uri)

        try:
            req = urllib.request.Request(uri)
            req.add_header('Authorization', 'token %s' % self.token)
            r = urllib.request.urlopen(req)
        except urllib.error.HTTPError as err:
            if err.code == 404:
                raise GithubFileNotFound('File %s is not available. Check the URL to ensure it really exists' % uri)
            else:
                raise

        return r.read().decode("utf-8")


Instruction:
Can you generate a brief explanation for the following Python 3 code
def measure_memory(cls, obj, seen=None):
        size = sys.getsizeof(obj)
        if seen is None:
            seen = set()
        obj_id = id(obj)
        if obj_id in seen:
            return 0
        # Important mark as seen *before* entering recursion to gracefully handle
        # self-referential objects
        seen.add(obj_id)
        if isinstance(obj, dict):
            size += sum([cls.measure_memory(v, seen) for v in obj.values()])
            size += sum([cls.measure_memory(k, seen) for k in obj.keys()])
        elif hasattr(obj, '__dict__'):
            size += cls.measure_memory(obj.__dict__, seen)
        elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes, bytearray)):
            size += sum([cls.measure_memory(i, seen) for i in obj])
        return size

Response: Recursively finds the size of objects and returns it.


Instruction:
Here you have a function in Python 3, explain what it does
def __feed_arthur(self):

        with self.ARTHUR_FEED_LOCK:

            # This is a expensive operation so don't do it always
            if (time.time() - self.ARTHUR_LAST_MEMORY_CHECK) > 5 * self.ARTHUR_LAST_MEMORY_CHECK_TIME:
                self.ARTHUR_LAST_MEMORY_CHECK = time.time()
                logger.debug("Measuring the memory used by the raw items dict ...")
                try:
                    memory_size = self.measure_memory(self.arthur_items) / (1024 * 1024)
                except RuntimeError as ex:
                    # During memory usage measure, other thread could change the dict
                    logger.warning("Can't get the memory used by the raw items dict: %s", ex)
                    memory_size = self.ARTHUR_LAST_MEMORY_SIZE
                self.ARTHUR_LAST_MEMORY_CHECK_TIME = time.time() - self.ARTHUR_LAST_MEMORY_CHECK
                logger.debug("Arthur items memory size: %0.2f MB (%is to check)",
                             memory_size, self.ARTHUR_LAST_MEMORY_CHECK_TIME)
                self.ARTHUR_LAST_MEMORY_SIZE = memory_size

            # Don't feed items from redis if the current python dict is
            # larger than ARTHUR_MAX_MEMORY_SIZE

            if self.ARTHUR_LAST_MEMORY_SIZE > self.ARTHUR_MAX_MEMORY_SIZE:
                logger.debug("Items queue full. Not collecting items from redis queue.")
                return

            logger.info("Collecting items from redis queue")

            db_url = self.config.get_conf()['es_collection']['redis_url']

            conn = redis.StrictRedis.from_url(db_url)
            logger.debug("Redis connection stablished with %s.", db_url)

            # Get and remove queued items in an atomic transaction
            pipe = conn.pipeline()
            # pipe.lrange(Q_STORAGE_ITEMS, 0, -1)
            pipe.lrange(Q_STORAGE_ITEMS, 0, self.ARTHUR_REDIS_ITEMS - 1)
            pipe.ltrim(Q_STORAGE_ITEMS, self.ARTHUR_REDIS_ITEMS, -1)
            items = pipe.execute()[0]

            for item in items:
                arthur_item = pickle.loads(item)
                if arthur_item['tag'] not in self.arthur_items:
                    self.arthur_items[arthur_item['tag']] = []
                self.arthur_items[arthur_item['tag']].append(arthur_item)

            for tag in self.arthur_items:
                if self.arthur_items[tag]:
                    logger.debug("Arthur items for %s: %i", tag, len(self.arthur_items[tag]))

Response: Feed items from arthur redis queue


Instruction:
How would you implement a function in Python 3 that
feeds items from arthur redis queue with backend data collected from arthur redis queue

Response:
def __feed_backend_arthur(self, repo):
        """ Feed Ocean with backend data collected from arthur redis queue"""

        # Always get pending items from arthur for all data sources
        self.__feed_arthur()

        tag = self.backend_tag(repo)

        logger.debug("Arthur items available for %s", self.arthur_items.keys())

        logger.debug("Getting arthur items for %s.", tag)

        if tag in self.arthur_items:
            logger.debug("Found items for %s.", tag)
            while self.arthur_items[tag]:
                yield self.arthur_items[tag].pop()


Instruction:
How would you implement a function in Python 3 that
creates the JSON for configuring arthur data

Response:
def __create_arthur_json(self, repo, backend_args):
        """ Create the JSON for configuring arthur to collect data

        https://github.com/grimoirelab/arthur#adding-tasks
        Sample for git:

        {
        "tasks": [
            {
                "task_id": "arthur.git",
                "backend": "git",
                "backend_args": {
                    "gitpath": "/tmp/arthur_git/",
                    "uri": "https://github.com/grimoirelab/arthur.git"
                },
                "category": "commit",
                "archive_args": {
                    "archive_path": '/tmp/test_archives',
                    "fetch_from_archive": false,
                    "archive_after": None
                },
                "scheduler_args": {
                    "delay": 10
                }
            }
        ]
        }
        """

        backend_args = self._compose_arthur_params(self.backend_section, repo)
        if self.backend_section == 'git':
            backend_args['gitpath'] = os.path.join(self.REPOSITORY_DIR, repo)
        backend_args['tag'] = self.backend_tag(repo)

        ajson = {"tasks": [{}]}
        # This is the perceval tag
        ajson["tasks"][0]['task_id'] = self.backend_tag(repo)
        ajson["tasks"][0]['backend'] = self.backend_section.split(":")[0]
        ajson["tasks"][0]['backend_args'] = backend_args
        ajson["tasks"][0]['category'] = backend_args['category']
        ajson["tasks"][0]['archive'] = {}
        ajson["tasks"][0]['scheduler'] = {"delay": self.ARTHUR_TASK_DELAY}
        # from-date or offset param must be added
        es_col_url = self._get_collection_url()
        es_index = self.conf[self.backend_section]['raw_index']
        # Get the last activity for the data source
        es = ElasticSearch(es_col_url, es_index)
        connector = get_connector_from_name(self.backend_section)

        klass = connector[0]  # Backend for the connector
        signature = inspect.signature(klass.fetch)

        last_activity = None
        filter_ = {"name": "tag", "value": backend_args['tag']}
        if 'from_date' in signature.parameters:
            last_activity = es.get_last_item_field('metadata__updated_on', [filter_])
            if last_activity:
                ajson["tasks"][0]['backend_args']['from_date'] = last_activity.isoformat()
        elif 'offset' in signature.parameters:
            last_activity = es.get_last_item_field('offset', [filter_])
            if last_activity:
                ajson["tasks"][0]['backend_args']['offset'] = last_activity

        if last_activity:
            logging.info("Getting raw item with arthur since %s", last_activity)

        return(ajson)


Instruction:
Write a Python 3 function for
getting the GitHub SHA for a file in the repository

Response:
def sha_github_file(cls, config, repo_file, repository_api, repository_branch):
        """ Return the GitHub SHA for a file in the repository """

        repo_file_sha = None

        cfg = config.get_conf()
        github_token = cfg['sortinghat']['identities_api_token']
        headers = {"Authorization": "token " + github_token}

        url_dir = repository_api + "/git/trees/" + repository_branch
        logger.debug("Gettting sha data from tree: %s", url_dir)
        raw_repo_file_info = requests.get(url_dir, headers=headers)
        raw_repo_file_info.raise_for_status()
        for rfile in raw_repo_file_info.json()['tree']:
            if rfile['path'] == repo_file:
                logger.debug("SHA found: %s, ", rfile["sha"])
                repo_file_sha = rfile["sha"]
                break

        return repo_file_sha


Instruction:
Write a Python 3 script to
get the uuid for a profile name

Response:
def __get_uuids_from_profile_name(self, profile_name):
        """ Get the uuid for a profile name """
        uuids = []

        with self.db.connect() as session:
            query = session.query(Profile).\
                filter(Profile.name == profile_name)
            profiles = query.all()
            if profiles:
                for p in profiles:
                    uuids.append(p.uuid)
        return uuids


Instruction:
Can you generate the documentation for the following Python 3 function
def micro_mordred(cfg_path, backend_sections, raw, arthur, identities, enrich, panels):

    config = Config(cfg_path)

    if raw:
        for backend in backend_sections:
            get_raw(config, backend, arthur)

    if identities:
        get_identities(config)

    if enrich:
        for backend in backend_sections:
            get_enrich(config, backend)

    if panels:
        get_panels(config)

Response: Execute the raw and enrich phases of a given backend section.


Instruction:
Create a Python 3 function for
executing the raw phase for a given backend section optionally using Arthur

Response:
def get_raw(config, backend_section, arthur):
    """Execute the raw phase for a given backend section, optionally using Arthur

    :param config: a Mordred config object
    :param backend_section: the backend section where the raw phase is executed
    :param arthur: if true, it enables Arthur to collect the raw data
    """

    if arthur:
        task = TaskRawDataArthurCollection(config, backend_section=backend_section)
    else:
        task = TaskRawDataCollection(config, backend_section=backend_section)

    TaskProjects(config).execute()
    try:
        task.execute()
        logging.info("Loading raw data finished!")
    except Exception as e:
        logging.error(str(e))
        sys.exit(-1)


Instruction:
Explain what the following Python 3 code does
def get_identities(config):

    TaskProjects(config).execute()
    task = TaskIdentitiesMerge(config)
    task.execute()
    logging.info("Merging identities finished!")

Response: Execute the merge identities phase


Instruction:
Can you tell what is the following Python 3 function doing
def get_enrich(config, backend_section):

    TaskProjects(config).execute()
    task = TaskEnrich(config, backend_section=backend_section)
    try:
        task.execute()
        logging.info("Loading enriched data finished!")
    except Exception as e:
        logging.error(str(e))
        sys.exit(-1)

Response: Execute the enrich phase for a given backend section


Instruction:
Write a Python 3 script for
executing the panels phase

Response:
def get_panels(config):
    """Execute the panels phase

    :param config: a Mordred config object
    """

    task = TaskPanels(config)
    task.execute()

    task = TaskPanelsMenu(config)
    task.execute()

    logging.info("Panels creation finished!")


Instruction:
Write a Python 3 script to
configure logging level output output

Response:
def config_logging(debug):
    """Config logging level output output"""

    if debug:
        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s')
        logging.debug("Debug mode activated")
    else:
        logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')


Instruction:
Explain what the following Python 3 code does
def get_params_parser():

    parser = argparse.ArgumentParser(add_help=False)

    parser.add_argument('-g', '--debug', dest='debug',
                        action='store_true',
                        help=argparse.SUPPRESS)
    parser.add_argument("--arthur", action='store_true', dest='arthur',
                        help="Enable arthur to collect raw data")
    parser.add_argument("--raw", action='store_true', dest='raw',
                        help="Activate raw task")
    parser.add_argument("--enrich", action='store_true', dest='enrich',
                        help="Activate enrich task")
    parser.add_argument("--identities", action='store_true', dest='identities',
                        help="Activate merge identities task")
    parser.add_argument("--panels", action='store_true', dest='panels',
                        help="Activate panels task")

    parser.add_argument("--cfg", dest='cfg_path',
                        help="Configuration file path")
    parser.add_argument("--backends", dest='backend_sections', default=[],
                        nargs='*', help="Backend sections to execute")

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    return parser

Response: Parse command line arguments


Instruction:
Can you tell what is the following Python 3 function doing
def get_params():

    parser = get_params_parser()
    args = parser.parse_args()

    if not args.raw and not args.enrich and not args.identities and not args.panels:
        print("No tasks enabled")
        sys.exit(1)

    return args

Response: Get params to execute the micro - mordred


Instruction:
How would you explain what the following Python 3 function does
def __kibiter_version(self):
        version = None

        es_url = self.conf['es_enrichment']['url']
        config_url = '.kibana/config/_search'
        url = urijoin(es_url, config_url)
        version = None
        try:
            res = self.grimoire_con.get(url)
            res.raise_for_status()
            version = res.json()['hits']['hits'][0]['_id']
            logger.debug("Kibiter version: %s", version)
        except requests.exceptions.HTTPError:
            logger.warning("Can not find Kibiter version")

        return version

Response: Get the kibiter vesion.


Instruction:
Can you write a function in Python 3 where it
uploads a panel to Elasticsearch if it does not exist yet.

Response:
def create_dashboard(self, panel_file, data_sources=None, strict=True):
        """Upload a panel to Elasticsearch if it does not exist yet.

        If a list of data sources is specified, upload only those
        elements (visualizations, searches) that match that data source.

        :param panel_file: file name of panel (dashobard) to upload
        :param data_sources: list of data sources
        :param strict: only upload a dashboard if it is newer than the one already existing
        """
        es_enrich = self.conf['es_enrichment']['url']
        kibana_url = self.conf['panels']['kibiter_url']

        mboxes_sources = set(['pipermail', 'hyperkitty', 'groupsio', 'nntp'])
        if data_sources and any(x in data_sources for x in mboxes_sources):
            data_sources = list(data_sources)
            data_sources.append('mbox')
        if data_sources and ('supybot' in data_sources):
            data_sources = list(data_sources)
            data_sources.append('irc')
        if data_sources and 'google_hits' in data_sources:
            data_sources = list(data_sources)
            data_sources.append('googlehits')
        if data_sources and 'stackexchange' in data_sources:
            # stackexchange is called stackoverflow in panels
            data_sources = list(data_sources)
            data_sources.append('stackoverflow')
        if data_sources and 'phabricator' in data_sources:
            data_sources = list(data_sources)
            data_sources.append('maniphest')

        try:
            import_dashboard(es_enrich, kibana_url, panel_file, data_sources=data_sources, strict=strict)
        except ValueError:
            logger.error("%s does not include release field. Not loading the panel.", panel_file)
        except RuntimeError:
            logger.error("Can not load the panel %s", panel_file)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def __upload_title(self, kibiter_major):

        if kibiter_major == "6":
            resource = ".kibana/doc/projectname"
            data = {"projectname": {"name": self.project_name}}
            mapping_resource = ".kibana/_mapping/doc"
            mapping = {"dynamic": "true"}

            url = urijoin(self.conf['es_enrichment']['url'], resource)
            mapping_url = urijoin(self.conf['es_enrichment']['url'],
                                  mapping_resource)

            logger.debug("Adding mapping for dashboard title")
            res = self.grimoire_con.put(mapping_url, data=json.dumps(mapping),
                                        headers=ES6_HEADER)
            try:
                res.raise_for_status()
            except requests.exceptions.HTTPError:
                logger.error("Couldn't create mapping for dashboard title.")
                logger.error(res.json())

            logger.debug("Uploading dashboard title")
            res = self.grimoire_con.post(url, data=json.dumps(data),
                                         headers=ES6_HEADER)
            try:
                res.raise_for_status()
            except requests.exceptions.HTTPError:
                logger.error("Couldn't create dashboard title.")
                logger.error(res.json())

Response: Upload to Kibiter the title for the dashboard.


Instruction:
Can you create a Python 3 function that
creates the dashboard menu for a Kibiter.

Response:
def __create_dashboard_menu(self, dash_menu, kibiter_major):
        """Create the menu definition to access the panels in a dashboard.

        :param          menu: dashboard menu to upload
        :param kibiter_major: major version of kibiter
        """
        logger.info("Adding dashboard menu")
        if kibiter_major == "6":
            menu_resource = ".kibana/doc/metadashboard"
            mapping_resource = ".kibana/_mapping/doc"
            mapping = {"dynamic": "true"}
            menu = {'metadashboard': dash_menu}
        else:
            menu_resource = ".kibana/metadashboard/main"
            mapping_resource = ".kibana/_mapping/metadashboard"
            mapping = {"dynamic": "true"}
            menu = dash_menu
        menu_url = urijoin(self.conf['es_enrichment']['url'],
                           menu_resource)

        mapping_url = urijoin(self.conf['es_enrichment']['url'],
                              mapping_resource)
        logger.debug("Adding mapping for metadashboard")
        res = self.grimoire_con.put(mapping_url, data=json.dumps(mapping),
                                    headers=ES6_HEADER)
        try:
            res.raise_for_status()
        except requests.exceptions.HTTPError:
            logger.error("Couldn't create mapping for Kibiter menu.")
        res = self.grimoire_con.post(menu_url, data=json.dumps(menu),
                                     headers=ES6_HEADER)
        try:
            res.raise_for_status()
        except requests.exceptions.HTTPError:
            logger.error("Couldn't create Kibiter menu.")
            logger.error(res.json())
            raise


Instruction:
Write a Python 3 script for
removing existing dashboard menu for kibiter.

Response:
def __remove_dashboard_menu(self, kibiter_major):
        """Remove existing menu for dashboard, if any.

        Usually, we remove the menu before creating a new one.

        :param kibiter_major: major version of kibiter
        """
        logger.info("Removing old dashboard menu, if any")
        if kibiter_major == "6":
            metadashboard = ".kibana/doc/metadashboard"
        else:
            metadashboard = ".kibana/metadashboard/main"
        menu_url = urijoin(self.conf['es_enrichment']['url'], metadashboard)
        self.grimoire_con.delete(menu_url)


Instruction:
How would you code a function in Python 3 to
get the menu entries from the panel definition

Response:
def __get_menu_entries(self, kibiter_major):
        """ Get the menu entries from the panel definition """
        menu_entries = []
        for entry in self.panels_menu:
            if entry['source'] not in self.data_sources:
                continue
            parent_menu_item = {
                'name': entry['name'],
                'title': entry['name'],
                'description': "",
                'type': "menu",
                'dashboards': []
            }
            for subentry in entry['menu']:
                try:
                    dash_name = get_dashboard_name(subentry['panel'])
                except FileNotFoundError:
                    logging.error("Can't open dashboard file %s", subentry['panel'])
                    continue
                # The name for the entry is in self.panels_menu
                child_item = {
                    "name": subentry['name'],
                    "title": subentry['name'],
                    "description": "",
                    "type": "entry",
                    "panel_id": dash_name
                }
                parent_menu_item['dashboards'].append(child_item)
            menu_entries.append(parent_menu_item)

        return menu_entries


Instruction:
How would you implement a function in Python 3 that
gets the dashboard menu for a given kibiter major number

Response:
def __get_dash_menu(self, kibiter_major):
        """Order the dashboard menu"""

        # omenu = OrderedDict()
        omenu = []
        # Start with Overview
        omenu.append(self.menu_panels_common['Overview'])

        # Now the data _getsources
        ds_menu = self.__get_menu_entries(kibiter_major)

        # Remove the kafka and community menus, they will be included at the end
        kafka_menu = None
        community_menu = None

        found_kafka = [pos for pos, menu in enumerate(ds_menu) if menu['name'] == KAFKA_NAME]
        if found_kafka:
            kafka_menu = ds_menu.pop(found_kafka[0])

        found_community = [pos for pos, menu in enumerate(ds_menu) if menu['name'] == COMMUNITY_NAME]
        if found_community:
            community_menu = ds_menu.pop(found_community[0])

        ds_menu.sort(key=operator.itemgetter('name'))
        omenu += ds_menu

        # If kafka and community are present add them before the Data Status and About
        if kafka_menu:
            omenu.append(kafka_menu)

        if community_menu:
            omenu.append(community_menu)

        # At the end Data Status, About
        omenu.append(self.menu_panels_common['Data Status'])
        omenu.append(self.menu_panels_common['About'])

        logger.debug("Menu for panels: %s", json.dumps(ds_menu, indent=4))
        return omenu


Instruction:
Here you have a function in Python 3, explain what it does
def compose_mbox(projects):
    mbox_archives = '/home/bitergia/mboxes'

    mailing_lists_projects = [project for project in projects if 'mailing_lists' in projects[project]]
    for mailing_lists in mailing_lists_projects:
        projects[mailing_lists]['mbox'] = []
        for mailing_list in projects[mailing_lists]['mailing_lists']:
            if 'listinfo' in mailing_list:
                name = mailing_list.split('listinfo/')[1]
            elif 'mailing-list' in mailing_list:
                name = mailing_list.split('mailing-list/')[1]
            else:
                name = mailing_list.split('@')[0]

            list_new = "%s %s/%s.mbox/%s.mbox" % (name, mbox_archives, name, name)
            projects[mailing_lists]['mbox'].append(list_new)

    return projects

Response:
Compose projects. json only for mbox but using the mailing_lists lists

    change to emfg - dev


Instruction:
Write a Python 3 function for
composing projects. json for gerrit but using the git lists change the git lists to the git root

Response:
def compose_gerrit(projects):
    """ Compose projects.json for gerrit, but using the git lists

    change: 'http://git.eclipse.org/gitroot/xwt/org.eclipse.xwt.git'
    to: 'git.eclipse.org_xwt/org.eclipse.xwt

    :param projects: projects.json
    :return: projects.json with gerrit
    """
    git_projects = [project for project in projects if 'git' in projects[project]]
    for project in git_projects:
        repos = [repo for repo in projects[project]['git'] if 'gitroot' in repo]
        if len(repos) > 0:
            projects[project]['gerrit'] = []
        for repo in repos:
            gerrit_project = repo.replace("http://git.eclipse.org/gitroot/", "")
            gerrit_project = gerrit_project.replace(".git", "")
            projects[project]['gerrit'].append("git.eclipse.org_" + gerrit_project)

    return projects


Instruction:
Write a Python 3 script to
compose projects. json for git

Response:
def compose_git(projects, data):
    """ Compose projects.json for git

    We need to replace '/c/' by '/gitroot/' for instance

    change: 'http://git.eclipse.org/c/xwt/org.eclipse.xwt.git'
    to: 'http://git.eclipse.org/gitroot/xwt/org.eclipse.xwt.git'

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with git
    """
    for p in [project for project in data if len(data[project]['source_repo']) > 0]:
        repos = []
        for url in data[p]['source_repo']:
            if len(url['url'].split()) > 1:  # Error at upstream the project 'tools.corrosion'
                repo = url['url'].split()[1].replace('/c/', '/gitroot/')
            else:
                repo = url['url'].replace('/c/', '/gitroot/')

            if repo not in repos:
                repos.append(repo)

        projects[p]['git'] = repos

    return projects


Instruction:
Can you create a Python 3 function that
composes projects. json for mailing lists at upstream

Response:
def compose_mailing_lists(projects, data):
    """ Compose projects.json for mailing lists

    At upstream has two different key for mailing list: 'mailings_lists' and 'dev_list'
    The key 'mailing_lists' is an array with mailing lists
    The key 'dev_list' is a dict with only one mailing list

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with mailing_lists
    """
    for p in [project for project in data if len(data[project]['mailing_lists']) > 0]:
        if 'mailing_lists' not in projects[p]:
            projects[p]['mailing_lists'] = []

        urls = [url['url'].replace('mailto:', '') for url in data[p]['mailing_lists'] if
                url['url'] not in projects[p]['mailing_lists']]
        projects[p]['mailing_lists'] += urls

    for p in [project for project in data if len(data[project]['dev_list']) > 0]:
        if 'mailing_lists' not in projects[p]:
            projects[p]['mailing_lists'] = []

        mailing_list = data[p]['dev_list']['url'].replace('mailto:', '')
        projects[p]['mailing_lists'].append(mailing_list)

    return projects


Instruction:
Here you have a function in Python 3, explain what it does
def compose_github(projects, data):
    for p in [project for project in data if len(data[project]['github_repos']) > 0]:
        if 'github' not in projects[p]:
            projects[p]['github'] = []

        urls = [url['url'] for url in data[p]['github_repos'] if
                url['url'] not in projects[p]['github']]
        projects[p]['github'] += urls

    return projects

Response: Compose projects. json with github


Instruction:
How would you explain what the following Python 3 function does
def compose_bugzilla(projects, data):
    for p in [project for project in data if len(data[project]['bugzilla']) > 0]:
        if 'bugzilla' not in projects[p]:
            projects[p]['bugzilla'] = []

        urls = [url['query_url'] for url in data[p]['bugzilla'] if
                url['query_url'] not in projects[p]['bugzilla']]
        projects[p]['bugzilla'] += urls

    return projects

Response: Compose the bugzilla project list for the project.


Instruction:
Make a summary of the following Python 3 code
def compose_title(projects, data):
    for project in data:
        projects[project] = {
            'meta': {
                'title': data[project]['title']
            }
        }
    return projects

Response: Compose the projects. json file only with the projects titles


Instruction:
Here you have a function in Python 3, explain what it does
def compose_projects_json(projects, data):
    projects = compose_git(projects, data)
    projects = compose_mailing_lists(projects, data)
    projects = compose_bugzilla(projects, data)
    projects = compose_github(projects, data)
    projects = compose_gerrit(projects)
    projects = compose_mbox(projects)

    return projects

Response: Compose projects. json with all data sources


Instruction:
Write a Python 3 function that can
execute autorefresh for studies

Response:
def __autorefresh_studies(self, cfg):
        """Execute autorefresh for areas of code study if configured"""

        if 'studies' not in self.conf[self.backend_section] or \
                'enrich_areas_of_code:git' not in self.conf[self.backend_section]['studies']:
            logger.debug("Not doing autorefresh for studies, Areas of Code study is not active.")
            return

        aoc_index = self.conf['enrich_areas_of_code:git'].get('out_index', GitEnrich.GIT_AOC_ENRICHED)

        # if `out_index` exists but has no value, use default
        if not aoc_index:
            aoc_index = GitEnrich.GIT_AOC_ENRICHED

        logger.debug("Autorefresh for Areas of Code study index: %s", aoc_index)

        es = Elasticsearch([self.conf['es_enrichment']['url']], timeout=100,
                           verify_certs=self._get_enrich_backend().elastic.requests.verify)

        if not es.indices.exists(index=aoc_index):
            logger.debug("Not doing autorefresh, index doesn't exist for Areas of Code study")
            return

        logger.debug("Doing autorefresh for Areas of Code study")

        # Create a GitEnrich backend tweaked to work with AOC index
        aoc_backend = GitEnrich(self.db_sh, None, cfg['projects']['projects_file'],
                                self.db_user, self.db_password, self.db_host)
        aoc_backend.mapping = None
        aoc_backend.roles = ['author']
        elastic_enrich = get_elastic(self.conf['es_enrichment']['url'],
                                     aoc_index, clean=False, backend=aoc_backend)
        aoc_backend.set_elastic(elastic_enrich)

        self.__autorefresh(aoc_backend, studies=True)


Instruction:
How would you code a function in Python 3 to
execute the studies configured for the current backend

Response:
def __studies(self, retention_time):
        """ Execute the studies configured for the current backend """

        cfg = self.config.get_conf()
        if 'studies' not in cfg[self.backend_section] or not \
           cfg[self.backend_section]['studies']:
            logger.debug('No studies for %s' % self.backend_section)
            return

        studies = [study for study in cfg[self.backend_section]['studies'] if study.strip() != ""]
        if not studies:
            logger.debug('No studies for %s' % self.backend_section)
            return

        logger.debug("Executing studies for %s: %s" % (self.backend_section, studies))
        time.sleep(2)  # Wait so enrichment has finished in ES
        enrich_backend = self._get_enrich_backend()
        ocean_backend = self._get_ocean_backend(enrich_backend)

        active_studies = []
        all_studies = enrich_backend.studies
        all_studies_names = [study.__name__ for study in enrich_backend.studies]

        # Time to check that configured studies are valid
        logger.debug("All studies in %s: %s", self.backend_section, all_studies_names)
        logger.debug("Configured studies %s", studies)
        cfg_studies_types = [study.split(":")[0] for study in studies]
        if not set(cfg_studies_types).issubset(set(all_studies_names)):
            logger.error('Wrong studies names for %s: %s', self.backend_section, studies)
            raise RuntimeError('Wrong studies names ', self.backend_section, studies)

        for study in enrich_backend.studies:
            if study.__name__ in cfg_studies_types:
                active_studies.append(study)

        enrich_backend.studies = active_studies
        print("Executing for %s the studies %s" % (self.backend_section,
              [study for study in studies]))

        studies_args = self.__load_studies()

        do_studies(ocean_backend, enrich_backend, studies_args, retention_time=retention_time)
        # Return studies to its original value
        enrich_backend.studies = all_studies


Instruction:
Implement a Python 3 function for
retaining the identities in SortingHat based on the retention_time value declared in the setup. cfg.

Response:
def retain_identities(self, retention_time):
        """Retain the identities in SortingHat based on the `retention_time`
        value declared in the setup.cfg.

        :param retention_time: maximum number of minutes wrt the current date to retain the SortingHat data
        """
        enrich_es = self.conf['es_enrichment']['url']
        sortinghat_db = self.db
        current_data_source = self.get_backend(self.backend_section)
        active_data_sources = self.config.get_active_data_sources()

        if retention_time is None:
            logger.debug("[identities retention] Retention policy disabled, no identities will be deleted.")
            return

        if retention_time <= 0:
            logger.debug("[identities retention] Retention time must be greater than 0.")
            return

        retain_identities(retention_time, enrich_es, sortinghat_db, current_data_source, active_data_sources)


Instruction:
Can you write a function in Python 3 where it
gets the list of repositories for a given backend section

Response:
def get_repos_by_backend_section(cls, backend_section, raw=True):
        """ return list with the repositories for a backend_section """
        repos = []
        projects = TaskProjects.get_projects()

        for pro in projects:
            if backend_section in projects[pro]:
                # if the projects.json doesn't contain the `unknown` project, add the repos in the bck section
                if cls.GLOBAL_PROJECT not in projects:
                    repos += projects[pro][backend_section]
                else:
                    # if the projects.json contains the `unknown` project
                    # in the case of the collection phase
                    if raw:
                        # if the current project is not `unknown`
                        if pro != cls.GLOBAL_PROJECT:
                            # if the bck section is not in the `unknown` project, add the repos in the bck section
                            if backend_section not in projects[cls.GLOBAL_PROJECT]:
                                repos += projects[pro][backend_section]
                            # if the backend section is in the `unknown` project,
                            # add the repo in the bck section under `unknown`
                            elif backend_section in projects[pro] and backend_section in projects[cls.GLOBAL_PROJECT]:
                                repos += projects[cls.GLOBAL_PROJECT][backend_section]
                        # if the current project is `unknown`
                        else:
                            # if the backend section is only in the `unknown` project,
                            # add the repo in the bck section under `unknown`
                            not_in_unknown = [projects[pro] for pro in projects if pro != cls.GLOBAL_PROJECT][0]
                            if backend_section not in not_in_unknown:
                                repos += projects[cls.GLOBAL_PROJECT][backend_section]
                    # in the case of the enrichment phase
                    else:
                        # if the current project is not `unknown`
                        if pro != cls.GLOBAL_PROJECT:
                            # if the bck section is not in the `unknown` project, add the repos in the bck section
                            if backend_section not in projects[cls.GLOBAL_PROJECT]:
                                repos += projects[pro][backend_section]
                            # if the backend section is in the `unknown` project, add the repos in the bck section
                            elif backend_section in projects[pro] and backend_section in projects[cls.GLOBAL_PROJECT]:
                                repos += projects[pro][backend_section]
                        # if the current project is `unknown`
                        else:
                            # if the backend section is only in the `unknown` project,
                            # add the repo in the bck section under `unknown`
                            not_in_unknown_prj = [projects[prj] for prj in projects if prj != cls.GLOBAL_PROJECT]
                            not_in_unknown_sections = list(set([section for prj in not_in_unknown_prj
                                                                for section in list(prj.keys())]))
                            if backend_section not in not_in_unknown_sections:
                                repos += projects[pro][backend_section]

        logger.debug("List of repos for %s: %s (raw=%s)", backend_section, repos, raw)

        # avoid duplicated repos
        repos = list(set(repos))

        return repos


Instruction:
Make a summary of the following Python 3 code
def convert_from_eclipse(self, eclipse_projects):

        projects = {}

        # We need the global project for downloading the full Bugzilla and Gerrit
        projects['unknown'] = {
            "gerrit": ["git.eclipse.org"],
            "bugzilla": ["https://bugs.eclipse.org/bugs/"]
        }

        projects = compose_title(projects, eclipse_projects)
        projects = compose_projects_json(projects, eclipse_projects)

        return projects

Response: Convert from eclipse projects format to grimoire projects json format


Instruction:
Here you have a function in Python 3, explain what it does
def general_params(cls):

        params = {}

        # GENERAL CONFIG
        params_general = {
            "general": {
                "min_update_delay": {
                    "optional": True,
                    "default": 60,
                    "type": int,
                    "description": "Short delay between tasks (collect, enrich ...)"
                },
                "update": {
                    "optional": False,
                    "default": False,
                    "type": bool,
                    "description": "Execute the tasks in loop"
                },
                "short_name": {
                    "optional": False,
                    "default": "Short name",
                    "type": str,
                    "description": "Short name of the project"
                },
                "debug": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Debug mode (logging mainly)"
                },
                "logs_dir": {
                    "optional": False,
                    "default": "logs",
                    "type": str,
                    "description": "Directory with the logs of sirmordred"
                },
                "log_handler": {
                    "optional": True,
                    "default": "file",
                    "type": str,
                    "description": "use rotate for rotating the logs automatically"
                },
                "log_max_bytes": {
                    "optional": True,
                    "default": 104857600,  # 100MB
                    "type": int,
                    "description": "Max number of bytes per log file"
                },
                "log_backup_count": {
                    "optional": True,
                    "default": 5,
                    "type": int,
                    "description": "Number of rotate logs files to preserve"
                },
                "bulk_size": {
                    "optional": True,
                    "default": 1000,
                    "type": int,
                    "description": "Number of items to write in Elasticsearch using bulk operations"
                },
                "scroll_size": {
                    "optional": True,
                    "default": 100,
                    "type": int,
                    "description": "Number of items to read from Elasticsearch when scrolling"
                },
                "aliases_file": {
                    "optional": True,
                    "default": ALIASES_JSON,
                    "type": str,
                    "description": "JSON file to define aliases for raw and enriched indexes"
                },
                "menu_file": {
                    "optional": True,
                    "default": MENU_YAML,
                    "type": str,
                    "description": "YAML file to define the menus to be shown in Kibiter"
                },
                "retention_time": {
                    "optional": True,
                    "default": None,
                    "type": int,
                    "description": "The maximum number of minutes wrt the current date to retain the data"
                }
            }
        }
        params_projects = {
            "projects": {
                "projects_file": {
                    "optional": True,
                    "default": PROJECTS_JSON,
                    "type": str,
                    "description": "Projects file path with repositories to be collected group by projects"
                },
                "projects_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Projects file URL"
                },
                "load_eclipse": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Load the projects from Eclipse"
                }
            }
        }

        params_phases = {
            "phases": {
                "collection": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Activate collection of items"
                },
                "enrichment": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Activate enrichment of items"
                },
                "identities": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Do the identities tasks"
                },
                "panels": {
                    "optional": False,
                    "default": True,
                    "type": bool,
                    "description": "Load panels, create alias and other tasks related"
                },
                "track_items": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Track specific items from a gerrit repository"
                },
                "report": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Generate the PDF report for a project (alpha)"
                }
            }
        }

        general_config_params = [params_general, params_projects, params_phases]

        for section_params in general_config_params:
            params.update(section_params)

        # Config provided by tasks
        params_collection = {
            "es_collection": {
                "password": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Password for connection to Elasticsearch"
                },
                "user": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "User for connection to Elasticsearch"
                },
                "url": {
                    "optional": False,
                    "default": "http://172.17.0.1:9200",
                    "type": str,
                    "description": "Elasticsearch URL"
                },
                "arthur": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Use arthur for collecting items from perceval"
                },
                "arthur_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "URL for the arthur service"
                },
                "redis_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "URL for the redis service"
                }
            }
        }

        params_enrichment = {
            "es_enrichment": {
                "url": {
                    "optional": False,
                    "default": "http://172.17.0.1:9200",
                    "type": str,
                    "description": "Elasticsearch URL"
                },
                "autorefresh": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "Execute the autorefresh of identities"
                },
                "autorefresh_interval": {
                    "optional": True,
                    "default": 2,
                    "type": int,
                    "description": "Set time interval (days) for autorefresh identities"
                },
                "user": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "User for connection to Elasticsearch"
                },
                "password": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Password for connection to Elasticsearch"
                }
            }
        }

        params_panels = {
            "panels": {
                "strict": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "Enable strict panels loading"
                },
                "kibiter_time_from": {
                    "optional": True,
                    "default": "now-90d",
                    "type": str,
                    "description": "Default time interval for Kibiter"
                },
                "kibiter_default_index": {
                    "optional": True,
                    "default": "git",
                    "type": str,
                    "description": "Default index pattern for Kibiter"
                },
                "kibiter_url": {
                    "optional": False,
                    "default": None,
                    "type": str,
                    "description": "Kibiter URL"
                },
                "kibiter_version": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Kibiter version"
                },
                "community": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "Enable community structure menu"
                },
                "kafka": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable kafka menu"
                },
                "github-repos": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable GitHub repo stats menu"
                },
                "gitlab-issues": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable GitLab issues menu"
                },
                "gitlab-merges": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable GitLab merge requests menu"
                },
                "mattermost": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Enable Mattermost menu"
                }
            }
        }

        params_report = {
            "report": {
                "start_date": {
                    "optional": False,
                    "default": "1970-01-01",
                    "type": str,
                    "description": "Start date for the report"
                },
                "end_date": {
                    "optional": False,
                    "default": "2100-01-01",
                    "type": str,
                    "description": "End date for the report"
                },
                "interval": {
                    "optional": False,
                    "default": "quarter",
                    "type": str,
                    "description": "Interval for the report"
                },
                "config_file": {
                    "optional": False,
                    "default": "report.cfg",
                    "type": str,
                    "description": "Config file for the report"
                },
                "data_dir": {
                    "optional": False,
                    "default": "report_data",
                    "type": str,
                    "description": "Directory in which to store the report data"
                },
                "filters": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "General filters to be applied to all queries"
                },
                "offset": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "Date offset to be applied to start and end"
                }
            }
        }

        params_sortinghat = {
            "sortinghat": {
                "affiliate": {
                    "optional": False,
                    "default": "True",
                    "type": bool,
                    "description": "Affiliate identities to organizations"
                },
                "unaffiliated_group": {
                    "optional": False,
                    "default": "Unknown",
                    "type": str,
                    "description": "Name of the organization for unaffiliated identities"
                },
                "matching": {
                    "optional": False,
                    "default": ["email"],
                    "type": list,
                    "description": "Algorithm for matching identities in Sortinghat"
                },
                "sleep_for": {
                    "optional": False,
                    "default": 3600,
                    "type": int,
                    "description": "Delay between task identities executions"
                },
                "database": {
                    "optional": False,
                    "default": "sortinghat_db",
                    "type": str,
                    "description": "Name of the Sortinghat database"
                },
                "host": {
                    "optional": False,
                    "default": "mariadb",
                    "type": str,
                    "description": "Host with the Sortinghat database"
                },
                "user": {
                    "optional": False,
                    "default": "root",
                    "type": str,
                    "description": "User to access the Sortinghat database"
                },
                "password": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "Password to access the Sortinghat database"
                },
                "autoprofile": {
                    "optional": False,
                    "default": ["customer", "git", "github"],
                    "type": list,
                    "description": "Order in which to get the identities information for filling the profile"
                },
                "load_orgs": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "deprecated": "Load organizations in Sortinghat database",
                    "description": ""
                },
                "identities_format": {
                    "optional": True,
                    "default": "sortinghat",
                    "type": str,
                    "description": "Format of the identities data to be loaded"
                },
                "strict_mapping": {
                    "optional": True,
                    "default": True,
                    "type": bool,
                    "description": "rigorous check of values in identities matching "
                                   "(i.e, well formed email addresses)"
                },
                "reset_on_load": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Unmerge and remove affiliations for all identities on load"
                },
                "orgs_file": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "File path with the organizations to be loaded in Sortinghat"
                },
                "identities_file": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "File path with the identities to be loaded in Sortinghat"
                },
                "identities_export_url": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "URL in which to export the identities in Sortinghat"
                },
                "identities_api_token": {
                    "optional": True,
                    "default": None,
                    "type": str,
                    "description": "API token for remote operation with GitHub and Gitlab"
                },
                "bots_names": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "Name of the identities to be marked as bots"
                },
                "no_bots_names": {
                    "optional": True,
                    "default": [],
                    "type": list,
                    "description": "Name of the identities to be unmarked as bots"
                },
                "autogender": {
                    "optional": True,
                    "default": False,
                    "type": bool,
                    "description": "Add gender to the profiles (executes autogender)"
                }
            }
        }

        params_track_items = {
            "track_items": {
                "project": {
                    "optional": False,
                    "default": "TrackProject",
                    "type": str,
                    "description": "Gerrit project to track"
                },
                "upstream_raw_es_url": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "URL with the file with the gerrit reviews to track"
                },
                "raw_index_gerrit": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "Name of the gerrit raw index"
                },
                "raw_index_git": {
                    "optional": False,
                    "default": "",
                    "type": str,
                    "description": "Name of the git raw index"
                }
            }
        }

        tasks_config_params = [params_collection, params_enrichment, params_panels,
                               params_report, params_sortinghat, params_track_items]

        for section_params in tasks_config_params:
            params.update(section_params)

        return params

Response: Define all the possible config params for the class


Instruction:
Write a Python 3 function for
changing a param in the config

Response:
def set_param(self, section, param, value):
        """ Change a param in the config """
        if section not in self.conf or param not in self.conf[section]:
            logger.error('Config section %s and param %s not exists', section, param)
        else:
            self.conf[section][param] = value


Instruction:
How would you code a function in Python 3 to
add new configuration to self. conf

Response:
def _add_to_conf(self, new_conf):
        """Add new configuration to self.conf.

        Adds configuration parameters in new_con to self.conf.
        If they already existed in conf, overwrite them.

        :param new_conf: new configuration, to add
        """

        for section in new_conf:
            if section not in self.conf:
                self.conf[section] = new_conf[section]
            else:
                for param in new_conf[section]:
                    self.conf[section][param] = new_conf[section][param]


Instruction:
How would you explain what the following Python 3 function does
def es_version(self, url):

        try:
            res = self.grimoire_con.get(url)
            res.raise_for_status()
            major = res.json()['version']['number'].split(".")[0]
        except Exception:
            logger.error("Error retrieving Elasticsearch version: " + url)
            raise
        return major

Response: Get the major version of Elasticsearch.


Instruction:
Can you create a Python 3 function that
executes the tasks_cls in the non - stop state.

Response:
def execute_nonstop_tasks(self, tasks_cls):
        """
            Just a wrapper to the execute_batch_tasks method
        """
        self.execute_batch_tasks(tasks_cls,
                                 self.conf['sortinghat']['sleep_for'],
                                 self.conf['general']['min_update_delay'], False)


Instruction:
Make a summary of the following Python 3 code
def execute_batch_tasks(self, tasks_cls, big_delay=0, small_delay=0, wait_for_threads=True):

        def _split_tasks(tasks_cls):
            """
            we internally distinguish between tasks executed by backend
            and tasks executed with no specific backend. """
            backend_t = []
            global_t = []
            for t in tasks_cls:
                if t.is_backend_task(t):
                    backend_t.append(t)
                else:
                    global_t.append(t)
            return backend_t, global_t

        backend_tasks, global_tasks = _split_tasks(tasks_cls)
        logger.debug('backend_tasks = %s' % (backend_tasks))
        logger.debug('global_tasks = %s' % (global_tasks))

        threads = []

        # stopper won't be set unless wait_for_threads is True
        stopper = threading.Event()

        # launching threads for tasks by backend
        if len(backend_tasks) > 0:
            repos_backend = self._get_repos_by_backend()
            for backend in repos_backend:
                # Start new Threads and add them to the threads list to complete
                t = TasksManager(backend_tasks, backend, stopper, self.config, small_delay)
                threads.append(t)
                t.start()

        # launch thread for global tasks
        if len(global_tasks) > 0:
            # FIXME timer is applied to all global_tasks, does it make sense?
            # All tasks are executed in the same thread sequentially
            gt = TasksManager(global_tasks, "Global tasks", stopper, self.config, big_delay)
            threads.append(gt)
            gt.start()
            if big_delay > 0:
                when = datetime.now() + timedelta(seconds=big_delay)
                when_str = when.strftime('%a, %d %b %Y %H:%M:%S %Z')
                logger.info("%s will be executed on %s" % (global_tasks, when_str))

        if wait_for_threads:
            time.sleep(1)  # Give enough time create and run all threads
            stopper.set()  # All threads must stop in the next iteration

        # Wait for all threads to complete
        for t in threads:
            t.join()

        # Checking for exceptions in threads to log them
        self.__check_queue_for_errors()

        logger.debug("[thread:main] All threads (and their tasks) are finished")

Response: Execute a list of tasks in a batch.


Instruction:
Make a summary of the following Python 3 code
def __execute_initial_load(self):

        if self.conf['phases']['panels']:
            tasks_cls = [TaskPanels, TaskPanelsMenu]
            self.execute_tasks(tasks_cls)
        if self.conf['phases']['identities']:
            tasks_cls = [TaskInitSortingHat]
            self.execute_tasks(tasks_cls)

        logger.info("Loading projects")
        tasks_cls = [TaskProjects]
        self.execute_tasks(tasks_cls)
        logger.info("Done")

        return

Response: Execute tasks that should be done just one time


Instruction:
Write a Python 3 script for
running the command line and returns a Result object.

Response:
def run(self, halt_on_nonzero=True, quiet=False, q=False, streaming=False):
        """
        After building your commands, call `run()` to have your code executed.
        """
        commands = str(self)
        if not (quiet or q):
            self._echo.cmd(commands)

        env = self._context[0].get('env', {}) if len(self._context) > 0 else os.environ
        executable = self.current_context.get('executable')
        try:
            process = subprocess.Popen(commands,
                                       bufsize=1,
                                       shell=True,
                                       env=env,
                                       stdin=subprocess.PIPE,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE,
                                       executable=executable,
                                       universal_newlines=True)
            result = Result(process, commands, self._context, streaming, halt_on_nonzero=halt_on_nonzero)

        except Exception as e:
            result = Result(None, commands, self._context, exception=e)
            result.dump_exception()
            if halt_on_nonzero:
                raise e

        finally:
            self.clear()

        return result


Instruction:
Can you tell what is the following Python 3 function doing
def validate_config(self):
        '''
        Validates the provided config to make sure all the required fields are
        there.
        '''
        # first ensure that all the required fields are there
        for key, key_config in self.params_map.items():
            if key_config['required']:
                if key not in self.config:
                    raise ValueError("Invalid Configuration! Required parameter '%s' was not provided to Sultan.")

        # second ensure that the fields that were pased were actually fields that
        # can be used
        for key in self.config.keys():
            if key not in self.params_map:
                raise ValueError("Invalid Configuration! The parameter '%s' provided is not used by Sultan!" % key)

Response: Validates the provided config to make sure all the required fields are in the config and that all the required fields are in the config that are in the params_map.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def stdout(self):
        if self._streaming:
            stdout = []
            while not self.__stdout.empty():
                try:
                    line = self.__stdout.get_nowait()
                    stdout.append(line)
                except:
                    pass
        else:
            stdout =  self.__stdout
        return stdout

Response: Converts the stdout string to a list.


Instruction:
Given the following Python 3 function, write the documentation
def stderr(self):
        if self._streaming:
            stderr = []
            while not self.__stderr.empty():
                try:
                    line = self.__stderr.get_nowait()
                    stderr.append(line)
                except:
                    pass
        else:
            stderr = self.__stderr
        return stderr

Response: Converts stderr string to a list.


Instruction:
Create a Python 3 function for
printing the stdout to console - if there is any stdout otherwise does nothing.

Response:
def print_stdout(self, always_print=False):
        """
        Prints the stdout to console - if there is any stdout, otherwise does nothing.
        :param always_print:   print the stdout, even if there is nothing in the buffer (default: false)
        """
        if self.__stdout or always_print:
            self.__echo.info("--{ STDOUT }---" + "-" * 100)
            self.__format_lines_info(self.stdout)
            self.__echo.info("---------------" + "-" * 100)


Instruction:
Can you write a function in Python 3 where it
prints the stderr to console - if there is any stdout otherwise does nothing.

Response:
def print_stderr(self, always_print=False):
        """
        Prints the stderr to console - if there is any stdout, otherwise does nothing.
        :param always_print:   print the stderr, even if there is nothing in the buffer (default: false)
        """
        if self.__stderr or always_print:
            self.__echo.critical("--{ STDERR }---" + "-" * 100)
            self.__format_lines_error(self.stderr)
            self.__echo.critical("---------------" + "-" * 100)


Instruction:
Write a Python 3 function for
printing the traceback to console.

Response:
def print_traceback(self, always_print=False):
        """
        Prints the traceback to console - if there is any traceback, otherwise does nothing.
        :param always_print:   print the traceback, even if there is nothing in the buffer (default: false)
        """
        if self._exception or always_print:
            self.__echo.critical("--{ TRACEBACK }" + "-" * 100)
            self.__format_lines_error(self.traceback)
            self.__echo.critical("---------------" + "-" * 100)


Instruction:
Make a summary of the following Python 3 code
def format(self, record):
        if isinstance(self.fmt, dict):
            self._fmt = self.fmt[record.levelname]
            if sys.version_info > (3, 2):
                # Update self._style because we've changed self._fmt
                # (code based on stdlib's logging.Formatter.__init__())
                if self.style not in logging._STYLES:
                    raise ValueError('Style must be one of: %s' % ','.join(
                        list(logging._STYLES.keys())))
                self._style = logging._STYLES[self.style][0](self._fmt)

        if sys.version_info > (2, 7):
            message = super(LevelFormatter, self).format(record)
        else:
            message = ColoredFormatter.format(self, record)

        return message

Response: Customize the message format based on the log level.


Instruction:
Can you generate the documentation for the following Python 3 function
def replace_print(fileobj=sys.stderr):
  printer = _Printer(fileobj)

  previous_stdout = sys.stdout
  sys.stdout = printer
  try:
    yield printer
  finally:
    sys.stdout = previous_stdout

Response: A context manager that replaces sys. stdout with a new printer.


Instruction:
Create a Python 3 function for
compacting a list of integers into a comma - separated string of intervals.

Response:
def compact_interval_string(value_list):
  """Compact a list of integers into a comma-separated string of intervals.

  Args:
    value_list: A list of sortable integers such as a list of numbers

  Returns:
    A compact string representation, such as "1-5,8,12-15"
  """

  if not value_list:
    return ''

  value_list.sort()

  # Start by simply building up a list of separate contiguous intervals
  interval_list = []
  curr = []
  for val in value_list:
    if curr and (val > curr[-1] + 1):
      interval_list.append((curr[0], curr[-1]))
      curr = [val]
    else:
      curr.append(val)

  if curr:
    interval_list.append((curr[0], curr[-1]))

  # For each interval collapse it down to "first, last" or just "first" if
  # if first == last.
  return ','.join([
      '{}-{}'.format(pair[0], pair[1]) if pair[0] != pair[1] else str(pair[0])
      for pair in interval_list
  ])


Instruction:
Can you implement a function in Python 3 that
gets a storage client using the provided credentials.

Response:
def _get_storage_service(credentials):
  """Get a storage client using the provided credentials or defaults."""
  if credentials is None:
    credentials = oauth2client.client.GoogleCredentials.get_application_default(
    )
  return discovery.build('storage', 'v1', credentials=credentials)


Instruction:
How would you code a function in Python 3 to
return True if we should retry False otherwise.

Response:
def _retry_storage_check(exception):
  """Return True if we should retry, False otherwise."""
  now = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
  print_error(
      '%s: Exception %s: %s' % (now, type(exception).__name__, str(exception)))
  return isinstance(exception, oauth2client.client.AccessTokenRefreshError)


Instruction:
Implement a function in Python 3 to
load a file from GCS.

Response:
def _load_file_from_gcs(gcs_file_path, credentials=None):
  """Load context from a text file in gcs.

  Args:
    gcs_file_path: The target file path; should have the 'gs://' prefix.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    The content of the text file as a string.
  """
  gcs_service = _get_storage_service(credentials)

  bucket_name, object_name = gcs_file_path[len('gs://'):].split('/', 1)
  request = gcs_service.objects().get_media(
      bucket=bucket_name, object=object_name)

  file_handle = io.BytesIO()
  downloader = MediaIoBaseDownload(file_handle, request, chunksize=1024 * 1024)
  done = False
  while not done:
    _, done = _downloader_next_chunk(downloader)
  filevalue = file_handle.getvalue()
  if not isinstance(filevalue, six.string_types):
    filevalue = filevalue.decode()
  return six.StringIO(filevalue)


Instruction:
Can you tell what is the following Python 3 function doing
def load_file(file_path, credentials=None):
  if file_path.startswith('gs://'):
    return _load_file_from_gcs(file_path, credentials)
  else:
    return open(file_path, 'r')

Response:
Loads a file from either local or gcs.
   .


Instruction:
Can you implement a function in Python 3 that
checks whether the file exists in GCS.

Response:
def _file_exists_in_gcs(gcs_file_path, credentials=None):
  """Check whether the file exists, in GCS.

  Args:
    gcs_file_path: The target file path; should have the 'gs://' prefix.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    True if the file's there.
  """
  gcs_service = _get_storage_service(credentials)

  bucket_name, object_name = gcs_file_path[len('gs://'):].split('/', 1)
  request = gcs_service.objects().get(
      bucket=bucket_name, object=object_name, projection='noAcl')
  try:
    request.execute()
    return True
  except errors.HttpError:
    return False


Instruction:
Here you have a function in Python 3, explain what it does
def file_exists(file_path, credentials=None):
  if file_path.startswith('gs://'):
    return _file_exists_in_gcs(file_path, credentials)
  else:
    return os.path.isfile(file_path)

Response: Checks whether the file exists on local disk or GCS.


Instruction:
Implement a Python 3 function for
checking whether there is a GCS object whose name starts with the prefix.

Response:
def _prefix_exists_in_gcs(gcs_prefix, credentials=None):
  """Check whether there is a GCS object whose name starts with the prefix.

  Since GCS doesn't actually have folders, this is how we check instead.

  Args:
    gcs_prefix: The path; should start with 'gs://'.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    True if the prefix matches at least one object in GCS.

  Raises:
    errors.HttpError: if it can't talk to the server
  """
  gcs_service = _get_storage_service(credentials)

  bucket_name, prefix = gcs_prefix[len('gs://'):].split('/', 1)
  # documentation in
  # https://cloud.google.com/storage/docs/json_api/v1/objects/list
  request = gcs_service.objects().list(
      bucket=bucket_name, prefix=prefix, maxResults=1)
  response = request.execute()
  return response.get('items', None)


Instruction:
Can you tell what is the following Python 3 function doing
def simple_pattern_exists_in_gcs(file_pattern, credentials=None):
  if '*' not in file_pattern:
    return _file_exists_in_gcs(file_pattern, credentials)
  if not file_pattern.startswith('gs://'):
    raise ValueError('file name must start with gs://')
  gcs_service = _get_storage_service(credentials)
  bucket_name, prefix = file_pattern[len('gs://'):].split('/', 1)
  if '*' in bucket_name:
    raise ValueError('Wildcards may not appear in the bucket name')
  # There is a '*' in prefix because we checked there's one in file_pattern
  # and there isn't one in bucket_name. Hence it must be in prefix.
  assert '*' in prefix
  prefix_no_wildcard = prefix[:prefix.index('*')]
  request = gcs_service.objects().list(
      bucket=bucket_name, prefix=prefix_no_wildcard)
  response = request.execute()
  if 'items' not in response:
    return False
  items_list = [i['name'] for i in response['items']]
  return any(fnmatch.fnmatch(i, prefix) for i in items_list)

Response: Returns True iff an object exists matching the input GCS pattern.


Instruction:
Here you have a function in Python 3, explain what it does
def outputs_are_present(outputs):
  # outputs are OutputFileParam (see param_util.py)

  # If outputs contain a pattern, then there is no way for `dsub` to verify
  # that *all* output is present. The best that `dsub` can do is to verify
  # that *some* output was created for each such parameter.
  for o in outputs:
    if not o.value:
      continue
    if o.recursive:
      if not folder_exists(o.value):
        return False
    else:
      if not simple_pattern_exists_in_gcs(o.value):
        return False
  return True

Response: True if each output contains at least one file or no output specified.