Instruction:
Implement a Python 3 function for
getting the issues from pagination

Response:
def issues(self, start=None):
        """Get the issues from pagination"""

        payload = self.__build_payload(size=self.items_per_page, operation=True, startdate=start)
        path = self.__get_url_project()
        return self.__fetch_items(path=path, payload=payload)


Instruction:
How would you code a function in Python 3 to
get the user data by URL

Response:
def user(self, user_name):
        """Get the user data by URL"""

        user = None

        if user_name in self._users:
            return self._users[user_name]

        url_user = self.__get_url("~" + user_name)

        logger.info("Getting info for %s" % (url_user))

        try:
            raw_user = self.__send_request(url_user)
            user = raw_user
        except requests.exceptions.HTTPError as e:
            if e.response.status_code in [404, 410]:
                logger.warning("Data is not available - %s", url_user)
                user = '{}'
            else:
                raise e

        self._users[user_name] = user

        return user


Instruction:
Can you generate a brief explanation for the following Python 3 code
def issue(self, issue_id):

        path = urijoin("bugs", str(issue_id))
        url_issue = self.__get_url(path)
        raw_text = self.__send_request(url_issue)

        return raw_text

Response: Get the issue data by its ID


Instruction:
Can you write a function in Python 3 where it
gets a collection list of a given issue

Response:
def issue_collection(self, issue_id, collection_name):
        """Get a collection list of a given issue"""

        path = urijoin("bugs", str(issue_id), collection_name)
        url_collection = self.__get_url(path)
        payload = {'ws.size': self.items_per_page, 'ws.start': 0, 'order_by': 'date_last_updated'}

        raw_items = self.__fetch_items(path=url_collection, payload=payload)

        return raw_items


Instruction:
Given the following Python 3 function, write the documentation
def __send_request(self, url, params=None):

        r = self.fetch(url, payload=params)
        return r.text

Response: Send a request to the server and return the response.


Instruction:
Write a Python 3 function that can
build the payload for the current session.

Response:
def __build_payload(self, size, operation=False, startdate=None):
        """Build payload"""

        payload = {
            'ws.size': size,
            'order_by': 'date_last_updated',
            'omit_duplicates': 'false',
            'status': ["New", "Incomplete", "Opinion", "Invalid", "Won't Fix",
                       "Expired", "Confirmed", "Triaged", "In Progress",
                       "Fix Committed", "Fix Released",
                       "Incomplete (with response)",
                       "Incomplete (without response)"]
        }

        if operation:
            payload['ws.op'] = 'searchTasks'
        if startdate:
            startdate = startdate.isoformat()
            payload['modified_since'] = startdate

        return payload


Instruction:
Write a Python 3 script for
fetching the items from Launchpad API using pagination

Response:
def __fetch_items(self, path, payload):
        """Return the items from Launchpad API using pagination"""

        page = 0  # current page
        url_next = path
        fetch_data = True

        while fetch_data:
            logger.debug("Fetching page: %i", page)

            try:
                raw_content = self.__send_request(url_next, payload)
                content = json.loads(raw_content)
            except requests.exceptions.HTTPError as e:
                if e.response.status_code in [410]:
                    logger.warning("Data is not available - %s", url_next)
                    raw_content = '{"total_size": 0, "start": 0, "entries": []}'
                    content = json.loads(raw_content)
                else:
                    raise e

            if 'next_collection_link' in content:
                url_next = content['next_collection_link']
                payload = None
            else:
                fetch_data = False

            yield raw_content
            page += 1


Instruction:
Can you tell what is the following Python 3 function doing
def fetch_items(self, category, **kwargs):
        from_date = kwargs['from_date']

        logger.info("Looking for messages from '%s' since %s",
                    self.uri, str(from_date))

        mailing_list = GroupsioClient(self.group_name, self.dirpath,
                                      self.api_token, self.verify)
        mailing_list.fetch()

        messages = self._fetch_and_parse_messages(mailing_list, from_date)

        for message in messages:
            yield message

        logger.info("Fetch process completed")

Response: Fetch the items from the group


Instruction:
Given the following Python 3 function, write the documentation
def fetch(self):
        logger.info("Downloading mboxes from '%s'", self.uri)
        logger.debug("Storing mboxes in '%s'", self.dirpath)

        if not os.path.exists(self.dirpath):
            os.makedirs(self.dirpath)

        group_id = self.__find_group_id()

        url = urijoin(GROUPSIO_API_URL, self.DOWNLOAD_ARCHIVES)
        payload = {'group_id': group_id}
        filepath = os.path.join(self.dirpath, MBOX_FILE)
        success = self._download_archive(url, payload, filepath)

        return success

Response: Fetch the mbox files from the remote archiver.


Instruction:
Can you tell what is the following Python 3 function doing
def subscriptions(self, per_page=PER_PAGE):
        url = urijoin(GROUPSIO_API_URL, self.GET_SUBSCRIPTIONS)
        logger.debug("Get groupsio paginated subscriptions from " + url)

        keep_fetching = True
        payload = {
            "limit": per_page
        }

        while keep_fetching:
            r = self.__fetch(url, payload)
            response_raw = r.json()
            subscriptions = response_raw['data']
            yield subscriptions

            total_subscriptions = response_raw['total_count']
            logger.debug("Subscriptions: %i/%i" % (response_raw['end_item'], total_subscriptions))

            payload['page_token'] = response_raw['next_page_token']
            keep_fetching = response_raw['has_more']

Response: Fetch the groupsio paginated subscriptions for a given token


Instruction:
Make a summary of the following Python 3 code
def __find_group_id(self):

        group_subscriptions = self.subscriptions(self.auth)

        for subscriptions in group_subscriptions:
            for sub in subscriptions:
                if sub['group_name'] == self.group_name:
                    return sub['group_id']

        msg = "Group id not found for group name %s" % self.group_name
        raise BackendError(cause=msg)

Response: Find the id of a group given its name by iterating on the list of subscriptions and returning the first one that matches the group name.


Instruction:
Create a Python 3 function for
fetching requests from groupsio API

Response:
def __fetch(self, url, payload):
        """Fetch requests from groupsio API"""

        r = requests.get(url, params=payload, auth=self.auth, verify=self.verify)
        try:
            r.raise_for_status()
        except requests.exceptions.HTTPError as e:
            raise e

        return r


Instruction:
Can you implement a function in Python 3 that
initializes mailing lists directory path

Response:
def _pre_init(self):
        """Initialize mailing lists directory path"""

        if not self.parsed_args.mboxes_path:
            base_path = os.path.expanduser('~/.perceval/mailinglists/')
            dirpath = os.path.join(base_path, GROUPSIO_URL, 'g', self.parsed_args.group_name)
        else:
            dirpath = self.parsed_args.mboxes_path

        setattr(self.parsed_args, 'dirpath', dirpath)


Instruction:
Implement a function in Python 3 to
return the Groupsio argument parser.

Response:
def setup_cmd_parser(cls):
        """Returns the Groupsio argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              token_auth=True)

        # Backend token is required
        action = parser.parser._option_string_actions['--api-token']
        action.required = True

        # Optional arguments
        group = parser.parser.add_argument_group('Groupsio arguments')
        group.add_argument('--mboxes-path', dest='mboxes_path',
                           help="Path where mbox files will be stored")
        group.add_argument('--no-verify', dest='verify',
                           action='store_false',
                           help="Value 'True' enable SSL verification")

        # Required arguments
        parser.parser.add_argument('group_name',
                                   help="Name of the group on Groups.io")

        return parser


Instruction:
Can you implement a function in Python 3 that
generates a UUID based on the given parameters.

Response:
def uuid(*args):
    """Generate a UUID based on the given parameters.

    The UUID will be the SHA1 of the concatenation of the values
    from the list. The separator bewteedn these values is ':'.
    Each value must be a non-empty string, otherwise, the function
    will raise an exception.

    :param *args: list of arguments used to generate the UUID

    :returns: a universal unique identifier

    :raises ValueError: when anyone of the values is not a string,
        is empty or `None`.
    """
    def check_value(v):
        if not isinstance(v, str):
            raise ValueError("%s value is not a string instance" % str(v))
        elif not v:
            raise ValueError("value cannot be None or empty")
        else:
            return v

    s = ':'.join(map(check_value, args))

    sha1 = hashlib.sha1(s.encode('utf-8', errors='surrogateescape'))
    uuid_sha1 = sha1.hexdigest()

    return uuid_sha1


Instruction:
Implement a function in Python 3 to
fetch items using the given backend. Generator to get items using the given backend class. When an archive manager is given, this function will store the fetched items in an `Archive`. If an exception is raised, this archive will be removed to avoid corrupted archives. The parameters needed to initialize the `backend` class and get the items are given using `backend_args` dict parameter. :param backend_class: backend class to fetch items :param backend_args: dict of arguments needed to fetch the items :param category: category of the items to retrieve. If None, it will use the default backend category :param filter_classified: remove classified fields from the resulting items :param manager: archive manager needed to store the items :returns: a generator of items

Response:
def fetch(backend_class, backend_args, category, filter_classified=False,
          manager=None):
    """Fetch items using the given backend.

    Generator to get items using the given backend class. When
    an archive manager is given, this function will store
    the fetched items in an `Archive`. If an exception is raised,
    this archive will be removed to avoid corrupted archives.

    The parameters needed to initialize the `backend` class and
    get the items are given using `backend_args` dict parameter.

    :param backend_class: backend class to fetch items
    :param backend_args: dict of arguments needed to fetch the items
    :param category: category of the items to retrieve.
       If None, it will use the default backend category
    :param filter_classified: remove classified fields from the resulting items
    :param manager: archive manager needed to store the items

    :returns: a generator of items
    """
    init_args = find_signature_parameters(backend_class.__init__,
                                          backend_args)
    archive = manager.create_archive() if manager else None
    init_args['archive'] = archive

    backend = backend_class(**init_args)

    if category:
        backend_args['category'] = category
    if filter_classified:
        backend_args['filter_classified'] = filter_classified

    fetch_args = find_signature_parameters(backend.fetch,
                                           backend_args)
    items = backend.fetch(**fetch_args)

    try:
        for item in items:
            yield item
    except Exception as e:
        if manager:
            archive_path = archive.archive_path
            manager.remove_archive(archive_path)
        raise e


Instruction:
Can you create a Python 3 function that
fetches items from an archive manager.

Response:
def fetch_from_archive(backend_class, backend_args, manager,
                       category, archived_after):
    """Fetch items from an archive manager.

    Generator to get the items of a category (previously fetched
    by the given backend class) from an archive manager. Only those
    items archived after the given date will be returned.

    The parameters needed to initialize `backend` and get the
    items are given using `backend_args` dict parameter.

    :param backend_class: backend class to retrive items
    :param backend_args: dict of arguments needed to retrieve the items
    :param manager: archive manager where the items will be retrieved
    :param category: category of the items to retrieve
    :param archived_after: return items archived after this date

    :returns: a generator of archived items
    """
    init_args = find_signature_parameters(backend_class.__init__,
                                          backend_args)
    backend = backend_class(**init_args)

    filepaths = manager.search(backend.origin,
                               backend.__class__.__name__,
                               category,
                               archived_after)

    for filepath in filepaths:
        backend.archive = Archive(filepath)
        items = backend.fetch_from_archive()

        try:
            for item in items:
                yield item
        except ArchiveError as e:
            logger.warning("Ignoring %s archive due to: %s", filepath, str(e))


Instruction:
Can you implement a function in Python 3 that
finds available backends under top_package and return a tuple of backend classes one with backendCommand classes and one with oneodata with backendCommand classes

Response:
def find_backends(top_package):
    """Find available backends.

    Look for the Perceval backends and commands under `top_package`
    and its sub-packages. When `top_package` defines a namespace,
    backends under that same namespace will be found too.

    :param top_package: package storing backends

    :returns: a tuple with two dicts: one with `Backend` classes and one
        with `BackendCommand` classes
    """
    candidates = pkgutil.walk_packages(top_package.__path__,
                                       prefix=top_package.__name__ + '.')

    modules = [name for _, name, is_pkg in candidates if not is_pkg]

    return _import_backends(modules)


Instruction:
Write a Python 3 script to
fetch items from a repository.

Response:
def fetch(self, category, filter_classified=False, **kwargs):
        """Fetch items from the repository.

        The method retrieves items from a repository.

        To removed classified fields from the resulting items, set
        the parameter `filter_classified`. Take into account this
        parameter is incompatible with archiving items. Raw client
        data are archived before any other process. Therefore,
        classified data  are stored within the archive. To prevent
        from possible data leaks or security issues when users do
        not need these fields, archiving and filtering are not
        compatible.

        :param category: the category of the items fetched
        :param filter_classified: remove classified fields from the resulting items
        :param kwargs: a list of other parameters (e.g., from_date, offset, etc.
        specific for each backend)

        :returns: a generator of items

        :raises BackendError: either when the category is not valid or
            'filter_classified' and 'archive' are active at the same time.
        """
        if category not in self.categories:
            cause = "%s category not valid for %s" % (category, self.__class__.__name__)
            raise BackendError(cause=cause)

        if filter_classified and self.archive:
            cause = "classified fields filtering is not compatible with archiving items"
            raise BackendError(cause=cause)

        if self.archive:
            self.archive.init_metadata(self.origin, self.__class__.__name__, self.version, category,
                                       kwargs)

        self.client = self._init_client()

        for item in self.fetch_items(category, **kwargs):
            if filter_classified:
                item = self.filter_classified_data(item)

            yield self.metadata(item, filter_classified=filter_classified)


Instruction:
Can you implement a function in Python 3 that
fetches the questions from an archive.

Response:
def fetch_from_archive(self):
        """Fetch the questions from an archive.

        It returns the items stored within an archive. If this method is called but
        no archive was provided, the method will raise a `ArchiveError` exception.

        :returns: a generator of items

        :raises ArchiveError: raised when an error occurs accessing an archive
        """
        if not self.archive:
            raise ArchiveError(cause="archive instance was not provided")

        self.client = self._init_client(from_archive=True)

        for item in self.fetch_items(self.archive.category, **self.archive.backend_params):
            yield self.metadata(item)


Instruction:
Implement a Python 3 function for
removing classified or confidential data from an item.

Response:
def filter_classified_data(self, item):
        """Remove classified or confidential data from an item.

        It removes those fields that contain data considered as classified.
        Classified fields are defined in `CLASSIFIED_FIELDS` class attribute.

        :param item: fields will be removed from this item

        :returns: the same item but with confidential data filtered
        """
        item_uuid = uuid(self.origin, self.metadata_id(item))

        logger.debug("Filtering classified data for item %s", item_uuid)

        for cf in self.CLASSIFIED_FIELDS:
            try:
                _remove_key_from_nested_dict(item, cf)
            except KeyError:
                logger.debug("Classified field '%s' not found for item %s; field ignored",
                             '.'.join(cf), item_uuid)

        logger.debug("Classified data filtered for item %s", item_uuid)

        return item


Instruction:
Can you create a Python 3 function that
adds metadata to an item.

Response:
def metadata(self, item, filter_classified=False):
        """Add metadata to an item.

        It adds metadata to a given item such as how and
        when it was fetched. The contents from the original item will
        be stored under the 'data' keyword.

        :param item: an item fetched by a backend
        :param filter_classified: sets if classified fields were filtered
        """
        item = {
            'backend_name': self.__class__.__name__,
            'backend_version': self.version,
            'perceval_version': __version__,
            'timestamp': datetime_utcnow().timestamp(),
            'origin': self.origin,
            'uuid': uuid(self.origin, self.metadata_id(item)),
            'updated_on': self.metadata_updated_on(item),
            'classified_fields_filtered': self.classified_fields if filter_classified else None,
            'category': self.metadata_category(item),
            'tag': self.tag,
            'data': item,
        }

        return item


Instruction:
Can you implement a function in Python 3 that
parses a list of arguments and return a parsed Namespace object.

Response:
def parse(self, *args):
        """Parse a list of arguments.

        Parse argument strings needed to run a backend command. The result
        will be a `argparse.Namespace` object populated with the values
        obtained after the validation of the parameters.

        :param args: argument strings

        :result: an object with the parsed values
        """
        parsed_args = self.parser.parse_args(args)

        # Category was not set, remove it
        if parsed_args.category is None:
            delattr(parsed_args, 'category')

        if self._from_date:
            parsed_args.from_date = str_to_datetime(parsed_args.from_date)
        if self._to_date and parsed_args.to_date:
            parsed_args.to_date = str_to_datetime(parsed_args.to_date)
        if self._archive and parsed_args.archived_since:
            parsed_args.archived_since = str_to_datetime(parsed_args.archived_since)

        if self._archive and parsed_args.fetch_archive and parsed_args.no_archive:
            raise AttributeError("fetch-archive and no-archive arguments are not compatible")
        if self._archive and parsed_args.fetch_archive and not parsed_args.category:
            raise AttributeError("fetch-archive needs a category to work with")

        # Set aliases
        for alias, arg in self.aliases.items():
            if (alias not in parsed_args) and (arg in parsed_args):
                value = getattr(parsed_args, arg, None)
                setattr(parsed_args, alias, value)

        return parsed_args


Instruction:
Explain what the following Python 3 code does
def _set_auth_arguments(self, basic_auth=True, token_auth=False):

        group = self.parser.add_argument_group('authentication arguments')

        if basic_auth:
            group.add_argument('-u', '--backend-user', dest='user',
                               help="backend user")
            group.add_argument('-p', '--backend-password', dest='password',
                               help="backend password")
        if token_auth:
            group.add_argument('-t', '--api-token', dest='api_token',
                               help="backend authentication token / API key")

Response: Activate authentication arguments parsing


Instruction:
Write a Python 3 script for
activating archive arguments parsing

Response:
def _set_archive_arguments(self):
        """Activate archive arguments parsing"""

        group = self.parser.add_argument_group('archive arguments')
        group.add_argument('--archive-path', dest='archive_path', default=None,
                           help="directory path to the archives")
        group.add_argument('--no-archive', dest='no_archive', action='store_true',
                           help="do not archive data")
        group.add_argument('--fetch-archive', dest='fetch_archive', action='store_true',
                           help="fetch data from the archives")
        group.add_argument('--archived-since', dest='archived_since', default='1970-01-01',
                           help="retrieve items archived since the given date")


Instruction:
Given the following Python 3 function, write the documentation
def _set_output_arguments(self):

        group = self.parser.add_argument_group('output arguments')
        group.add_argument('-o', '--output', type=argparse.FileType('w'),
                           dest='outfile', default=sys.stdout,
                           help="output file")
        group.add_argument('--json-line', dest='json_line', action='store_true',
                           help="produce a JSON line for each output item")

Response: Activate output arguments parsing


Instruction:
Write a Python 3 function for
fetching and write items from the given base - class.

Response:
def run(self):
        """Fetch and write items.

        This method runs the backend to fetch the items from the given
        origin. Items are converted to JSON objects and written to the
        defined output.

        If `fetch-archive` parameter was given as an argument during
        the inizialization of the instance, the items will be retrieved
        using the archive manager.
        """
        backend_args = vars(self.parsed_args)
        category = backend_args.pop('category', None)
        filter_classified = backend_args.pop('filter_classified', False)
        archived_since = backend_args.pop('archived_since', None)

        if self.archive_manager and self.parsed_args.fetch_archive:
            items = fetch_from_archive(self.BACKEND, backend_args,
                                       self.archive_manager,
                                       category,
                                       archived_since)
        else:
            items = fetch(self.BACKEND, backend_args, category,
                          filter_classified=filter_classified,
                          manager=self.archive_manager)

        try:
            for item in items:
                if self.json_line:
                    obj = json.dumps(item, separators=(',', ':'), sort_keys=True)
                else:
                    obj = json.dumps(item, indent=4, sort_keys=True)
                self.outfile.write(obj)
                self.outfile.write('\n')
        except IOError as e:
            raise RuntimeError(str(e))
        except Exception as e:
            raise RuntimeError(str(e))


Instruction:
Explain what the following Python 3 code does
def _initialize_archive(self):

        if 'archive_path' not in self.parsed_args:
            manager = None
        elif self.parsed_args.no_archive:
            manager = None
        else:
            if not self.parsed_args.archive_path:
                archive_path = os.path.expanduser(ARCHIVES_DEFAULT_PATH)
            else:
                archive_path = self.parsed_args.archive_path

            manager = ArchiveManager(archive_path)

        self.archive_manager = manager

Response: Initialize the archive based on the parsed parameters


Instruction:
Can you implement a function in Python 3 that
fetches the items from the backend

Response:
def fetch_items(self, category, **kwargs):
        """Fetch the messages

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']

        logger.info("Looking for messages from '%s' on '%s' since %s",
                    self.uri, self.dirpath, str(from_date))

        mailing_list = MailingList(self.uri, self.dirpath)

        messages = self._fetch_and_parse_messages(mailing_list, from_date)

        for message in messages:
            yield message

        logger.info("Fetch process completed")


Instruction:
Explain what the following Python 3 code does
def metadata_updated_on(item):
        ts = item[MBox.DATE_FIELD]
        ts = str_to_datetime(ts)

        return ts.timestamp()

Response: Extracts the update time from a MBox item.


Instruction:
Write a Python 3 function that can
parse a mbox file and return an iterator of dictionaries. Each one of this contains an email message. Each one of this contains an email message.

Response:
def parse_mbox(filepath):
        """Parse a mbox file.

        This method parses a mbox file and returns an iterator of dictionaries.
        Each one of this contains an email message.

        :param filepath: path of the mbox to parse

        :returns : generator of messages; each message is stored in a
            dictionary of type `requests.structures.CaseInsensitiveDict`
        """
        mbox = _MBox(filepath, create=False)

        for msg in mbox:
            message = message_to_dict(msg)
            yield message


Instruction:
Here you have a function in Python 3, explain what it does
def _fetch_and_parse_messages(self, mailing_list, from_date):

        from_date = datetime_to_utc(from_date)

        nmsgs, imsgs, tmsgs = (0, 0, 0)

        for mbox in mailing_list.mboxes:
            tmp_path = None

            try:
                tmp_path = self._copy_mbox(mbox)

                for message in self.parse_mbox(tmp_path):
                    tmsgs += 1

                    if not self._validate_message(message):
                        imsgs += 1
                        continue

                    # Ignore those messages sent before the given date
                    dt = str_to_datetime(message[MBox.DATE_FIELD])

                    if dt < from_date:
                        logger.debug("Message %s sent before %s; skipped",
                                     message['unixfrom'], str(from_date))
                        tmsgs -= 1
                        continue

                    # Convert 'CaseInsensitiveDict' to dict
                    message = self._casedict_to_dict(message)

                    nmsgs += 1
                    logger.debug("Message %s parsed", message['unixfrom'])

                    yield message
            except (OSError, EOFError) as e:
                logger.warning("Ignoring %s mbox due to: %s", mbox.filepath, str(e))
            except Exception as e:
                if tmp_path and os.path.exists(tmp_path):
                    os.remove(tmp_path)
                raise e
            finally:
                if tmp_path and os.path.exists(tmp_path):
                    os.remove(tmp_path)

        logger.info("Done. %s/%s messages fetched; %s ignored",
                    nmsgs, tmsgs, imsgs)

Response: Fetch and parse the messages from a mailing list from a given date.


Instruction:
Can you generate the documentation for the following Python 3 function
def _copy_mbox(self, mbox):

        tmp_path = tempfile.mktemp(prefix='perceval_')

        with mbox.container as f_in:
            with open(tmp_path, mode='wb') as f_out:
                for l in f_in:
                    f_out.write(l)
        return tmp_path

Response: Copy the contents of a mbox to a temporary file


Instruction:
Make a summary of the following Python 3 code
def _validate_message(self, message):

        # This check is "case insensitive" because we're
        # using 'CaseInsensitiveDict' from requests.structures
        # module to store the contents of a message.
        if self.MESSAGE_ID_FIELD not in message:
            logger.warning("Field 'Message-ID' not found in message %s; ignoring",
                           message['unixfrom'])
            return False

        if not message[self.MESSAGE_ID_FIELD]:
            logger.warning("Field 'Message-ID' is empty in message %s; ignoring",
                           message['unixfrom'])
            return False

        if self.DATE_FIELD not in message:
            logger.warning("Field 'Date' not found in message %s; ignoring",
                           message['unixfrom'])
            return False

        if not message[self.DATE_FIELD]:
            logger.warning("Field 'Date' is empty in message %s; ignoring",
                           message['unixfrom'])
            return False

        try:
            str_to_datetime(message[self.DATE_FIELD])
        except InvalidDateError:
            logger.warning("Invalid date %s in message %s; ignoring",
                           message[self.DATE_FIELD], message['unixfrom'])
            return False

        return True

Response: Check if the given message has the mandatory fields.


Instruction:
Can you tell what is the following Python 3 function doing
def _casedict_to_dict(self, message):
        message_id = message.pop(self.MESSAGE_ID_FIELD)
        date = message.pop(self.DATE_FIELD)

        msg = {k: v for k, v in message.items()}
        msg[self.MESSAGE_ID_FIELD] = message_id
        msg[self.DATE_FIELD] = date

        return msg

Response: Convert a message in CaseInsensitiveDict to dict.


Instruction:
Here you have a function in Python 3, explain what it does
def get_message(self, key):

        start, stop = self._lookup(key)
        self._file.seek(start)
        from_line = self._file.readline().replace(mailbox.linesep, b'')
        string = self._file.read(stop - self._file.tell())
        msg = self._message_factory(string.replace(mailbox.linesep, b'\n'))

        try:
            msg.set_from(from_line[5:].decode('ascii'))
            return msg
        except UnicodeDecodeError:
            pass

        try:
            msg.set_from(from_line[5:].decode('utf-8'))
        except UnicodeDecodeError:
            msg.set_from(from_line[5:].decode('iso-8859-1'))

        return msg

Response: Return a Message representation or raise a KeyError.


Instruction:
Can you generate the documentation for the following Python 3 function
def mboxes(self):
        archives = []

        if os.path.isfile(self.dirpath):
            try:
                archives.append(MBoxArchive(self.dirpath))
            except OSError as e:
                logger.warning("Ignoring %s mbox due to: %s", self.dirpath, str(e))
        else:
            for root, _, files in os.walk(self.dirpath):
                for filename in sorted(files):
                    try:
                        location = os.path.join(root, filename)
                        archives.append(MBoxArchive(location))
                    except OSError as e:
                        logger.warning("Ignoring %s mbox due to: %s", filename, str(e))
        return archives

Response: Returns the list of. MBoxArchive objects managed by this mailing list.


Instruction:
Can you write a function in Python 3 where it
fetches commits from a Git repository or a Git log file.

Response:
def fetch(self, category=CATEGORY_COMMIT, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME,
              branches=None, latest_items=False, no_update=False):
        """Fetch commits.

        The method retrieves from a Git repository or a log file
        a list of commits. Commits are returned in the same order
        they were obtained.

        When `from_date` parameter is given it returns items commited
        since the given date.

        The list of `branches` is a list of strings, with the names of
        the branches to fetch. If the list of branches is empty, no
        commit is fetched. If the list of branches is None, all commits
        for all branches will be fetched.

        The parameter `latest_items` returns only those commits which
        are new since the last time this method was called.

        The parameter `no_update` returns all commits without performing
        an update of the repository before.

        Take into account that `from_date` and `branches` are ignored
        when the commits are fetched from a Git log file or when
        `latest_items` flag is set.

        The class raises a `RepositoryError` exception when an error
        occurs accessing the repository.

        :param category: the category of items to fetch
        :param from_date: obtain commits newer than a specific date
            (inclusive)
        :param to_date: obtain commits older than a specific date
        :param branches: names of branches to fetch from (default: None)
        :param latest_items: sync with the repository to fetch only the
            newest commits
        :param no_update: if enabled, don't update the repo with the latest changes

        :returns: a generator of commits
        """
        if not from_date:
            from_date = DEFAULT_DATETIME
        if not to_date:
            to_date = DEFAULT_LAST_DATETIME

        kwargs = {
            'from_date': from_date,
            'to_date': to_date,
            'branches': branches,
            'latest_items': latest_items,
            'no_update': no_update
        }
        items = super().fetch(category, **kwargs)

        return items


Instruction:
Can you generate the documentation for the following Python 3 function
def fetch_items(self, category, **kwargs):
        from_date = kwargs['from_date']
        to_date = kwargs['to_date']
        branches = kwargs['branches']
        latest_items = kwargs['latest_items']
        no_update = kwargs['no_update']

        ncommits = 0

        try:
            if os.path.isfile(self.gitpath):
                commits = self.__fetch_from_log()
            else:
                commits = self.__fetch_from_repo(from_date, to_date, branches,
                                                 latest_items, no_update)

            for commit in commits:
                yield commit
                ncommits += 1
        except EmptyRepositoryError:
            pass

        logger.info("Fetch process completed: %s commits fetched",
                    ncommits)

Response:
Fetch the commits

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items


Instruction:
Given the following Python 3 function, write the documentation
def parse_git_log_from_file(filepath):
        with open(filepath, 'r', errors='surrogateescape',
                  newline=os.linesep) as f:
            parser = GitParser(f)

            for commit in parser.parse():
                yield commit

Response: Parse a Git log file and return an iterator of commits. Each one of this contains a commit. The iterator of commits contains a dictionary containing the commit ID and the commit ID of the commit.


Instruction:
Write a Python 3 function that can
initialize repositories directory path

Response:
def _pre_init(self):
        """Initialize repositories directory path"""

        if self.parsed_args.git_log:
            git_path = self.parsed_args.git_log
        elif not self.parsed_args.git_path:
            base_path = os.path.expanduser('~/.perceval/repositories/')
            processed_uri = self.parsed_args.uri.lstrip('/')
            git_path = os.path.join(base_path, processed_uri) + '-git'
        else:
            git_path = self.parsed_args.git_path

        setattr(self.parsed_args, 'gitpath', git_path)


Instruction:
Can you generate the documentation for the following Python 3 function
def setup_cmd_parser(cls):

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              to_date=True)

        # Optional arguments
        group = parser.parser.add_argument_group('Git arguments')
        group.add_argument('--branches', dest='branches',
                           nargs='+', type=str, default=None,
                           help="Fetch commits only from these branches")

        # Mutual exclusive parameters
        exgroup = group.add_mutually_exclusive_group()
        exgroup.add_argument('--git-path', dest='git_path',
                             help="Path where the Git repository will be cloned")
        exgroup.add_argument('--git-log', dest='git_log',
                             help="Path to the Git log file")

        exgroup_fetch = group.add_mutually_exclusive_group()
        exgroup_fetch.add_argument('--latest-items', dest='latest_items',
                                   action='store_true',
                                   help="Fetch latest commits added to the repository")
        exgroup_fetch.add_argument('--no-update', dest='no_update',
                                   action='store_true',
                                   help="Fetch all commits without updating the repository")

        # Required arguments
        parser.parser.add_argument('uri',
                                   help="URI of the Git log repository")

        return parser

Response: Returns the Git argument parser.


Instruction:
Can you tell what is the following Python 3 function doing
def parse(self):

        for line in self.stream:
            line = line.rstrip('\n')
            parsed = False
            self.nline += 1

            while not parsed:
                parsed = self.handlers[self.state](line)

                if self.state == self.COMMIT and self.commit:
                    commit = self._build_commit()
                    logger.debug("Commit %s parsed", commit['commit'])
                    yield commit

        # Return the last commit, if any
        if self.commit:
            commit = self._build_commit()
            logger.debug("Commit %s parsed", commit['commit'])
            yield commit

Response: Parse the Git log stream.


Instruction:
Can you generate the documentation for the following Python 3 function
def __get_old_filepath(self, f):
        i = f.find('{')
        j = f.find('}')

        if i > -1 and j > -1:
            prefix = f[0:i]
            inner = f[i + 1:f.find(' => ', i)]
            suffix = f[j + 1:]
            return prefix + inner + suffix
        elif ' => ' in f:
            return f.split(' => ')[0]
        else:
            return f

Response: Get the old filepath of a moved or renamed file.


Instruction:
Can you implement a function in Python 3 that
clones a Git repository.

Response:
def clone(cls, uri, dirpath):
        """Clone a Git repository.

        Make a bare copy of the repository stored in `uri` into `dirpath`.
        The repository would be either local or remote.

        :param uri: URI of the repository
        :param dirtpath: directory where the repository will be cloned

        :returns: a `GitRepository` class having cloned the repository

        :raises RepositoryError: when an error occurs cloning the given
            repository
        """
        cmd = ['git', 'clone', '--bare', uri, dirpath]
        env = {
            'LANG': 'C',
            'HOME': os.getenv('HOME', '')
        }

        cls._exec(cmd, env=env)

        logger.debug("Git %s repository cloned into %s",
                     uri, dirpath)

        return cls(uri, dirpath)


Instruction:
Implement a function in Python 3 to
return the total number of objects packed and unpacked on the repository.

Response:
def count_objects(self):
        """Count the objects of a repository.

        The method returns the total number of objects (packed and unpacked)
        available on the repository.

        :raises RepositoryError: when an error occurs counting the objects
            of a repository
        """
        cmd_count = ['git', 'count-objects', '-v']

        outs = self._exec(cmd_count, cwd=self.dirpath, env=self.gitenv)
        outs = outs.decode('utf-8', errors='surrogateescape').rstrip()

        try:
            cobjs = {k: v for k, v in (x.split(': ') for x in outs.split('\n'))}
            nobjs = int(cobjs['count']) + int(cobjs['in-pack'])
        except KeyError as e:
            error = "unable to parse 'count-objects' output; reason: '%s' entry not found" \
                % e.args[0]
            raise RepositoryError(cause=error)
        except ValueError as e:
            error = "unable to parse 'count-objects' output; reason: %s" % str(e)
            raise RepositoryError(cause=error)

        logger.debug("Git %s repository has %s objects",
                     self.uri, str(nobjs))

        return nobjs


Instruction:
Can you generate the documentation for the following Python 3 function
def is_detached(self):
        cmd_sym = ['git', 'symbolic-ref', 'HEAD']

        try:
            self._exec(cmd_sym, cwd=self.dirpath, env=self.gitenv)
        except RepositoryError as e:
            if e.msg.find("ref HEAD is not a symbolic ref") == -1:
                raise e
            return True
        else:
            return False

Response: Check if the repository is detached.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def update(self):
        cmd_update = ['git', 'fetch', 'origin', '+refs/heads/*:refs/heads/*', '--prune']
        self._exec(cmd_update, cwd=self.dirpath, env=self.gitenv)

        logger.debug("Git %s repository updated into %s",
                     self.uri, self.dirpath)

Response: Update the local copy of the current repository with the remote repository.


Instruction:
Write a Python 3 script for
keeping the repository in sync.

Response:
def sync(self):
        """Keep the repository in sync.

        This method will synchronize the repository with its 'origin',
        fetching newest objects and updating references. It uses low
        level commands which allow to keep track of which things
        have changed in the repository.

        The method also returns a list of hashes related to the new
        commits fetched during the process.

        :returns: list of new commits

        :raises RepositoryError: when an error occurs synchronizing
            the repository
        """
        pack_name, refs = self._fetch_pack()

        if pack_name:
            commits = self._read_commits_from_pack(pack_name)
        else:
            commits = []
            logger.debug("Git repository %s (%s) does not have any new object",
                         self.uri, self.dirpath)

        self._update_references(refs)

        logger.debug("Git repository %s (%s) is synced",
                     self.uri, self.dirpath)

        return commits


Instruction:
Create a Python 3 function for
reading the list commits from the repository.

Response:
def rev_list(self, branches=None):
        """Read the list commits from the repository

        The list of branches is a list of strings, with the names of the
        branches to fetch. If the list of branches is empty, no commit
        is fetched. If the list of branches is None, all commits
        for all branches will be fetched.

        The method returns the Git rev-list of the repository using the
        following options:

            git rev-list --topo-order

        :param branches: names of branches to fetch from (default: None)

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs executing the command
        """
        if self.is_empty():
            logger.warning("Git %s repository is empty; unable to get the rev-list",
                           self.uri)
            raise EmptyRepositoryError(repository=self.uri)

        cmd_rev_list = ['git', 'rev-list', '--topo-order']

        if branches is None:
            cmd_rev_list.extend(['--branches', '--tags', '--remotes=origin'])
        elif len(branches) == 0:
            cmd_rev_list.extend(['--branches', '--tags', '--max-count=0'])
        else:
            branches = ['refs/heads/' + branch for branch in branches]
            cmd_rev_list.extend(branches)

        for line in self._exec_nb(cmd_rev_list, cwd=self.dirpath, env=self.gitenv):
            yield line.rstrip('\n')

        logger.debug("Git rev-list fetched from %s repository (%s)",
                     self.uri, self.dirpath)


Instruction:
Make a summary of the following Python 3 code
def log(self, from_date=None, to_date=None, branches=None, encoding='utf-8'):
        if self.is_empty():
            logger.warning("Git %s repository is empty; unable to get the log",
                           self.uri)
            raise EmptyRepositoryError(repository=self.uri)

        cmd_log = ['git', 'log', '--reverse', '--topo-order']
        cmd_log.extend(self.GIT_PRETTY_OUTPUT_OPTS)

        if from_date:
            dt = from_date.strftime("%Y-%m-%d %H:%M:%S %z")
            cmd_log.append('--since=' + dt)

        if to_date:
            dt = to_date.strftime("%Y-%m-%d %H:%M:%S %z")
            cmd_log.append('--until=' + dt)

        if branches is None:
            cmd_log.extend(['--branches', '--tags', '--remotes=origin'])
        elif len(branches) == 0:
            cmd_log.append('--max-count=0')
        else:
            branches = ['refs/heads/' + branch for branch in branches]
            cmd_log.extend(branches)

        for line in self._exec_nb(cmd_log, cwd=self.dirpath, env=self.gitenv):
            yield line

        logger.debug("Git log fetched from %s repository (%s)",
                     self.uri, self.dirpath)

Response: Read the commit log of the current object.


Instruction:
Create a Python 3 function to
show the data of a set of commits.

Response:
def show(self, commits=None, encoding='utf-8'):
        """Show the data of a set of commits.

        The method returns the output of Git show command for a
        set of commits using the following options:

            git show --raw --numstat --pretty=fuller --decorate=full
                --parents -M -C -c [<commit>...<commit>]

        When the list of commits is empty, the command will return
        data about the last commit, like the default behaviour of
        `git show`.

        :param commits: list of commits to show data
        :param encoding: encode the output using this format

        :returns: a generator where each item is a line from the show output

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs fetching the show output
        """
        if self.is_empty():
            logger.warning("Git %s repository is empty; unable to run show",
                           self.uri)
            raise EmptyRepositoryError(repository=self.uri)

        if commits is None:
            commits = []

        cmd_show = ['git', 'show']
        cmd_show.extend(self.GIT_PRETTY_OUTPUT_OPTS)
        cmd_show.extend(commits)

        for line in self._exec_nb(cmd_show, cwd=self.dirpath, env=self.gitenv):
            yield line

        logger.debug("Git show fetched from %s repository (%s)",
                     self.uri, self.dirpath)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _fetch_pack(self):

        def prepare_refs(refs):
            return [ref.hash.encode('utf-8') for ref in refs
                    if not ref.refname.endswith('^{}')]

        def determine_wants(refs):
            remote_refs = prepare_refs(self._discover_refs(remote=True))
            local_refs = prepare_refs(self._discover_refs())
            wants = [ref for ref in remote_refs if ref not in local_refs]
            return wants

        client, repo_path = dulwich.client.get_transport_and_path(self.uri)
        repo = dulwich.repo.Repo(self.dirpath)
        fd = io.BytesIO()

        local_refs = self._discover_refs()
        graph_walker = _GraphWalker(local_refs)

        result = client.fetch_pack(repo_path,
                                   determine_wants,
                                   graph_walker,
                                   fd.write)
        refs = [GitRef(ref_hash.decode('utf-8'), ref_name.decode('utf-8'))
                for ref_name, ref_hash in result.refs.items()]

        if len(fd.getvalue()) > 0:
            fd.seek(0)
            pack = repo.object_store.add_thin_pack(fd.read, None)
            pack_name = pack.name().decode('utf-8')
        else:
            pack_name = None

        return (pack_name, refs)

Response: Fetch changes and store them in a pack.


Instruction:
How would you implement a function in Python 3 that
reads the commits of a pack.

Response:
def _read_commits_from_pack(self, packet_name):
        """Read the commits of a pack."""

        filepath = 'objects/pack/pack-' + packet_name

        cmd_verify_pack = ['git', 'verify-pack', '-v', filepath]

        outs = self._exec(cmd_verify_pack, cwd=self.dirpath, env=self.gitenv)
        outs = outs.decode('utf-8', errors='surrogateescape').rstrip()

        lines = [line.split(' ') for line in outs.split('\n')]

        # Commits usually come in the pack ordered from newest to oldest
        commits = [parts[0] for parts in lines if parts[1] == 'commit']
        commits.reverse()

        return commits


Instruction:
Can you create a Python 3 function that
updates references removing old ones.

Response:
def _update_references(self, refs):
        """Update references removing old ones."""

        new_refs = [ref.refname for ref in refs]

        # Delete old references
        for old_ref in self._discover_refs():
            if not old_ref.refname.startswith('refs/heads/'):
                continue
            if old_ref.refname in new_refs:
                continue
            self._update_ref(old_ref, delete=True)

        # Update new references
        for new_ref in refs:
            refname = new_ref.refname

            if refname.endswith('^{}'):
                logger.debug("Annotated tag %s ignored for updating in sync process",
                             refname)
                continue
            elif not refname.startswith('refs/heads/') and not refname.startswith('refs/tags/'):
                logger.debug("Reference %s not needed; ignored for updating in sync process",
                             refname)
                continue
            else:
                self._update_ref(new_ref)

        # Prune repository to remove old branches
        cmd = ['git', 'remote', 'prune', 'origin']
        self._exec(cmd, cwd=self.dirpath, env=self.gitenv)


Instruction:
Can you generate the documentation for the following Python 3 function
def _discover_refs(self, remote=False):

        if remote:
            cmd_refs = ['git', 'ls-remote', '-h', '-t', '--exit-code', 'origin']
            sep = '\t'
            ignored_error_codes = [2]
        else:
            # Check first whether the local repo is empty;
            # Running 'show-ref' in empty repos gives an error
            if self.is_empty():
                raise EmptyRepositoryError(repository=self.uri)

            cmd_refs = ['git', 'show-ref', '--heads', '--tags']
            sep = ' '
            ignored_error_codes = [1]

        # Error codes returned when no matching refs (i.e, no heads
        # or tags) are found in a repository will be ignored. Otherwise,
        # the full process would fail for those situations.
        outs = self._exec(cmd_refs, cwd=self.dirpath,
                          env=self.gitenv,
                          ignored_error_codes=ignored_error_codes)
        outs = outs.decode('utf-8', errors='surrogateescape').rstrip()
        outs = outs.split('\n') if outs else []

        refs = []

        for line in outs:
            data = line.split(sep)
            ref = GitRef(data[0], data[1])
            refs.append(ref)

        return refs

Response: Get the current list of local or remote refs.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _exec_nb(self, cmd, cwd=None, env=None, encoding='utf-8'):
        self.failed_message = None

        logger.debug("Running command %s (cwd: %s, env: %s)",
                     ' '.join(cmd), cwd, str(env))

        try:
            self.proc = subprocess.Popen(cmd,
                                         stdout=subprocess.PIPE,
                                         stderr=subprocess.PIPE,
                                         cwd=cwd,
                                         env=env)
            err_thread = threading.Thread(target=self._read_stderr,
                                          kwargs={'encoding': encoding},
                                          daemon=True)
            err_thread.start()
            for line in self.proc.stdout:
                yield line.decode(encoding, errors='surrogateescape')
            err_thread.join()

            self.proc.communicate()
            self.proc.stdout.close()
            self.proc.stderr.close()
        except OSError as e:
            err_thread.join()
            raise RepositoryError(cause=str(e))

        if self.proc.returncode != 0:
            cause = "git command - %s (return code: %d)" % \
                (self.failed_message, self.proc.returncode)
            raise RepositoryError(cause=cause)

Response: Execute a command with a non blocking call.


Instruction:
Can you tell what is the following Python 3 function doing
def _read_stderr(self, encoding='utf-8'):
        for line in self.proc.stderr:
            err_line = line.decode(encoding, errors='surrogateescape')

            if self.proc.returncode != 0:
                # If the subprocess didn't finish successfully, we expect
                # the last line in stderr to provide the cause
                if self.failed_message is not None:
                    # We had a message, there is a newer line, print it
                    logger.debug("Git log stderr: " + self.failed_message)
                self.failed_message = err_line
            else:
                # The subprocess is successfully up to now, print the line
                logger.debug("Git log stderr: " + err_line)

Response: Reads the stderr of the git log and writes it to self. failed_message.


Instruction:
Can you implement a function in Python 3 that
executes a command in the directory set by cwd.

Response:
def _exec(cmd, cwd=None, env=None, ignored_error_codes=None,
              encoding='utf-8'):
        """Run a command.

        Execute `cmd` command in the directory set by `cwd`. Environment
        variables can be set using the `env` dictionary. The output
        data is returned as encoded bytes.

        Commands which their returning status codes are non-zero will
        be treated as failed. Error codes considered as valid can be
        ignored giving them in the `ignored_error_codes` list.

        :returns: the output of the command as encoded bytes

        :raises RepositoryError: when an error occurs running the command
        """
        if ignored_error_codes is None:
            ignored_error_codes = []

        logger.debug("Running command %s (cwd: %s, env: %s)",
                     ' '.join(cmd), cwd, str(env))

        try:
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    cwd=cwd, env=env)
            (outs, errs) = proc.communicate()
        except OSError as e:
            raise RepositoryError(cause=str(e))

        if proc.returncode != 0 and proc.returncode not in ignored_error_codes:
            err = errs.decode(encoding, errors='surrogateescape')
            cause = "git command - %s" % err
            raise RepositoryError(cause=cause)
        else:
            logger.debug(errs.decode(encoding, errors='surrogateescape'))

        return outs


Instruction:
Can you generate the documentation for the following Python 3 function
def fetch(self, category=CATEGORY_TWEET, since_id=None, max_id=None,
              geocode=None, lang=None,
              include_entities=True, tweets_type=TWEET_TYPE_MIXED):
        """Fetch the tweets from the server.

        This method fetches tweets from the TwitterSearch API published in the last seven days.

        :param category: the category of items to fetch
        :param since_id: if not null, it returns results with an ID greater than the specified ID
        :param max_id: when it is set or if not None, it returns results with an ID less than the specified ID
        :param geocode: if enabled, returns tweets by users located at latitude,longitude,"mi"|"km"
        :param lang: if enabled, restricts tweets to the given language, given by an ISO 639-1 code
        :param include_entities: if disabled, it excludes entities node
        :param tweets_type: type of tweets returned. Default is “mixed”, others are "recent" and "popular"

        :returns: a generator of tweets
        """
        kwargs = {"since_id": since_id,
                  "max_id": max_id,
                  "geocode": geocode,
                  "lang": lang,
                  "include_entities": include_entities,
                  "result_type": tweets_type}
        items = super().fetch(category, **kwargs)

        return items

Response: Fetch the tweets from the TwitterSearch API.


Instruction:
Create a Python 3 function to
fetch the items from the backend

Response:
def fetch_items(self, category, **kwargs):
        """Fetch the tweets

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        since_id = kwargs['since_id']
        max_id = kwargs['max_id']
        geocode = kwargs['geocode']
        lang = kwargs['lang']
        entities = kwargs['include_entities']
        tweets_type = kwargs['result_type']

        logger.info("Fetching tweets %s from %s to %s",
                    self.query, str(since_id),
                    str(max_id) if max_id else '--')

        tweets_ids = []
        min_date = None
        max_date = None
        group_tweets = self.client.tweets(self.query, since_id=since_id, max_id=max_id, geocode=geocode,
                                          lang=lang, include_entities=entities, result_type=tweets_type)

        for tweets in group_tweets:
            for i in range(len(tweets)):
                tweet = tweets[i]
                tweets_ids.append(tweet['id'])

                if tweets[-1] == tweet:
                    min_date = str_to_datetime(tweets[-1]['created_at'])

                if tweets[0] == tweet and not max_date:
                    max_date = str_to_datetime(tweets[0]['created_at'])

                yield tweet

        logger.info("Fetch process completed: %s (unique %s) tweets fetched, from %s to %s",
                    len(tweets_ids), len(list(set(tweets_ids))), min_date, max_date)


Instruction:
Create a Python 3 function to
initialize the client object.

Response:
def _init_client(self, from_archive=False):
        """Init client"""

        return TwitterClient(self.api_token, self.max_items,
                             self.sleep_for_rate, self.min_rate_to_sleep, self.sleep_time,
                             self.archive, from_archive)


Instruction:
Write a Python 3 script for
fetching tweets for a given query between since_id and max_id.

Response:
def tweets(self, query, since_id=None, max_id=None, geocode=None, lang=None,
               include_entities=True, result_type=TWEET_TYPE_MIXED):
        """Fetch tweets for a given query between since_id and max_id.

        :param query: query to fetch tweets
        :param since_id: if not null, it returns results with an ID greater than the specified ID
        :param max_id: if not null, it returns results with an ID less than the specified ID
        :param geocode: if enabled, returns tweets by users located at latitude,longitude,"mi"|"km"
        :param lang: if enabled, restricts tweets to the given language, given by an ISO 639-1 code
        :param include_entities: if disabled, it excludes entities node
        :param result_type: type of tweets returned. Default is “mixed”, others are "recent" and "popular"

        :returns: a generator of tweets
        """
        resource = self.base_url
        params = {'q': query,
                  'count': self.max_items}

        if since_id:
            params['since_id'] = since_id

        if max_id:
            params['max_id'] = max_id

        if geocode:
            params['geocode'] = geocode

        if lang:
            params['lang'] = lang

        params['include_entities'] = include_entities
        params['result_type'] = result_type

        while True:
            raw_tweets = self._fetch(resource, params=params)
            tweets = json.loads(raw_tweets)

            if not tweets['statuses']:
                break

            params['max_id'] = tweets['statuses'][-1]['id'] - 1
            yield tweets['statuses']


Instruction:
Can you generate the documentation for the following Python 3 function
def _fetch(self, url, params):
        if not self.from_archive:
            self.sleep_for_rate_limit()

        headers = {'Authorization': 'Bearer ' + self.api_key}
        r = self.fetch(url, payload=params, headers=headers)

        if not self.from_archive:
            self.update_rate_limit(r)

        return r.text

Response:
Fetch a resource. A resource. a
        method to iterate over the contents of a
        type of resource and parameters.


Instruction:
Here you have a function in Python 3, explain what it does
def setup_cmd_parser(cls):

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              token_auth=True,
                                              archive=True)

        # Backend token is required
        action = parser.parser._option_string_actions['--api-token']
        action.required = True

        # Meetup options
        group = parser.parser.add_argument_group('Twitter arguments')
        group.add_argument('--max-items', dest='max_items',
                           type=int, default=MAX_ITEMS,
                           help="Maximum number of items requested on the same query")
        group.add_argument('--no-entities', dest='include_entities',
                           action='store_false',
                           help=" Exclude entities node")
        group.add_argument('--geo-code', dest='geocode',
                           help="Select tweets by users located at latitude,longitude,radius")
        group.add_argument('--lang', dest='lang',
                           help="Select tweets to the given language in ISO 639-1 code")
        group.add_argument('--tweets-type', dest='tweets_type', default=TWEET_TYPE_MIXED,
                           help="Type of tweets returned. Default is 'mixed', others are 'recent' and 'popular'")
        group.add_argument('--sleep-for-rate', dest='sleep_for_rate',
                           action='store_true',
                           help="sleep for getting more rate")
        group.add_argument('--min-rate-to-sleep', dest='min_rate_to_sleep',
                           default=MIN_RATE_LIMIT, type=int,
                           help="sleep until reset when the rate limit reaches this value")
        group.add_argument('--sleep-time', dest='sleep_time',
                           default=SLEEP_TIME, type=int,
                           help="minimun sleeping time to avoid too many request exception")

        # Required arguments
        parser.parser.add_argument('query',
                                   help="Search query including operators, max 500 chars")

        return parser

Response: Returns the Twitter argument parser.


Instruction:
How would you implement a function in Python 3 that
fetches data from Google API.

Response:
def fetch(self, category=CATEGORY_HITS):
        """Fetch data from Google API.

        The method retrieves a list of hits for some
        given keywords using the Google API.

        :param category: the category of items to fetch

        :returns: a generator of data
        """
        kwargs = {}
        items = super().fetch(category, **kwargs)

        return items


Instruction:
Can you generate a brief explanation for the following Python 3 code
def fetch_items(self, category, **kwargs):
        logger.info("Fetching data for '%s'", self.keywords)

        hits_raw = self.client.hits(self.keywords)
        hits = self.__parse_hits(hits_raw)

        yield hits

        logger.info("Fetch process completed")

Response: Fetch items from Google


Instruction:
Can you write a function in Python 3 where it
parses the hits returned by the Google Search API

Response:
def __parse_hits(self, hit_raw):
        """Parse the hits returned by the Google Search API"""

        # Create the soup and get the desired div
        bs_result = bs4.BeautifulSoup(hit_raw, 'html.parser')
        hit_string = bs_result.find("div", id="resultStats").text

        # Remove commas or dots
        hit_string = hit_string.replace(',', u'')
        hit_string = hit_string.replace('.', u'')

        fetched_on = datetime_utcnow().timestamp()
        id_args = self.keywords[:]
        id_args.append(str(fetched_on))

        hits_json = {
            'fetched_on': fetched_on,
            'id': uuid(*id_args),
            'keywords': self.keywords,
            'type': 'googleSearchHits'
        }

        if not hit_string:
            logger.warning("No hits for %s", self.keywords)
            hits_json['hits'] = 0

            return hits_json

        str_hits = re.search(r'\d+', hit_string).group(0)
        hits = int(str_hits)
        hits_json['hits'] = hits

        return hits_json


Instruction:
Write a Python 3 script for
fetching information about a list of keywords.

Response:
def hits(self, keywords):
        """Fetch information about a list of keywords."""

        if len(keywords) == 1:
            query_str = keywords[0]
        else:
            query_str = ' '.join([k for k in keywords])

        logger.info("Fetching hits for '%s'", query_str)
        params = {'q': query_str}

        # Make the request
        req = self.fetch(GOOGLE_SEARCH_URL, payload=params)

        return req.text


Instruction:
How would you explain what the following Python 3 function does
def fetch(self, category=CATEGORY_ISSUE, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME):
        if not from_date:
            from_date = DEFAULT_DATETIME
        if not to_date:
            to_date = DEFAULT_LAST_DATETIME

        from_date = datetime_to_utc(from_date)
        to_date = datetime_to_utc(to_date)

        kwargs = {
            'from_date': from_date,
            'to_date': to_date
        }
        items = super().fetch(category, **kwargs)

        return items

Response: Fetch the issues and pull requests from a GitHub repository.


Instruction:
Write a Python 3 function that can
fetch the items from the backend

Response:
def fetch_items(self, category, **kwargs):
        """Fetch the items (issues or pull_requests)

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']
        to_date = kwargs['to_date']

        if category == CATEGORY_ISSUE:
            items = self.__fetch_issues(from_date, to_date)
        elif category == CATEGORY_PULL_REQUEST:
            items = self.__fetch_pull_requests(from_date, to_date)
        else:
            items = self.__fetch_repo_info()

        return items


Instruction:
Create a Python 3 function for
extracting the update time from a GitHub item.

Response:
def metadata_updated_on(item):
        """Extracts the update time from a GitHub item.

        The timestamp used is extracted from 'updated_at' field.
        This date is converted to UNIX timestamp format. As GitHub
        dates are in UTC the conversion is straightforward.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        if "forks_count" in item:
            return item['fetched_on']
        else:
            ts = item['updated_at']
            ts = str_to_datetime(ts)

            return ts.timestamp()


Instruction:
Write a Python 3 script to
extract the category from a GitHub item.

Response:
def metadata_category(item):
        """Extracts the category from a GitHub item.

        This backend generates two types of item which are
        'issue' and 'pull_request'.
        """

        if "base" in item:
            category = CATEGORY_PULL_REQUEST
        elif "forks_count" in item:
            category = CATEGORY_REPO
        else:
            category = CATEGORY_ISSUE

        return category


Instruction:
Can you write a function in Python 3 where it
fetches the issues from the API

Response:
def __fetch_issues(self, from_date, to_date):
        """Fetch the issues"""

        issues_groups = self.client.issues(from_date=from_date)

        for raw_issues in issues_groups:
            issues = json.loads(raw_issues)
            for issue in issues:

                if str_to_datetime(issue['updated_at']) > to_date:
                    return

                self.__init_extra_issue_fields(issue)
                for field in TARGET_ISSUE_FIELDS:

                    if not issue[field]:
                        continue

                    if field == 'user':
                        issue[field + '_data'] = self.__get_user(issue[field]['login'])
                    elif field == 'assignee':
                        issue[field + '_data'] = self.__get_issue_assignee(issue[field])
                    elif field == 'assignees':
                        issue[field + '_data'] = self.__get_issue_assignees(issue[field])
                    elif field == 'comments':
                        issue[field + '_data'] = self.__get_issue_comments(issue['number'])
                    elif field == 'reactions':
                        issue[field + '_data'] = \
                            self.__get_issue_reactions(issue['number'], issue['reactions']['total_count'])

                yield issue


Instruction:
How would you implement a function in Python 3 that
fetches the pull requests from the server

Response:
def __fetch_pull_requests(self, from_date, to_date):
        """Fetch the pull requests"""

        raw_pulls = self.client.pulls(from_date=from_date)
        for raw_pull in raw_pulls:
            pull = json.loads(raw_pull)

            if str_to_datetime(pull['updated_at']) > to_date:
                return

            self.__init_extra_pull_fields(pull)
            for field in TARGET_PULL_FIELDS:

                if not pull[field]:
                    continue

                if field == 'user':
                    pull[field + '_data'] = self.__get_user(pull[field]['login'])
                elif field == 'merged_by':
                    pull[field + '_data'] = self.__get_user(pull[field]['login'])
                elif field == 'review_comments':
                    pull[field + '_data'] = self.__get_pull_review_comments(pull['number'])
                elif field == 'requested_reviewers':
                    pull[field + '_data'] = self.__get_pull_requested_reviewers(pull['number'])
                elif field == 'commits':
                    pull[field + '_data'] = self.__get_pull_commits(pull['number'])

            yield pull


Instruction:
Write a Python 3 script to
get repo info about stars watchers and forks

Response:
def __fetch_repo_info(self):
        """Get repo info about stars, watchers and forks"""

        raw_repo = self.client.repo()
        repo = json.loads(raw_repo)

        fetched_on = datetime_utcnow()
        repo['fetched_on'] = fetched_on.timestamp()

        yield repo


Instruction:
Here you have a function in Python 3, explain what it does
def __get_issue_reactions(self, issue_number, total_count):

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.issue_reactions(issue_number)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

Response: Get the issue reactions


Instruction:
Here you have a function in Python 3, explain what it does
def __get_issue_comment_reactions(self, comment_id, total_count):

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.issue_comment_reactions(comment_id)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

Response: Get reactions on issue comments


Instruction:
Explain what the following Python 3 code does
def __get_pull_requested_reviewers(self, pr_number):

        requested_reviewers = []
        group_requested_reviewers = self.client.pull_requested_reviewers(pr_number)

        for raw_requested_reviewers in group_requested_reviewers:
            group_requested_reviewers = json.loads(raw_requested_reviewers)

            for requested_reviewer in group_requested_reviewers['users']:
                user_data = self.__get_user(requested_reviewer['login'])
                requested_reviewers.append(user_data)

        return requested_reviewers

Response: Get pull request requested reviewers


Instruction:
Explain what the following Python 3 code does
def __get_pull_commits(self, pr_number):

        hashes = []
        group_pull_commits = self.client.pull_commits(pr_number)

        for raw_pull_commits in group_pull_commits:

            for commit in json.loads(raw_pull_commits):
                commit_hash = commit['sha']
                hashes.append(commit_hash)

        return hashes

Response: Get the list of commit hashes for a pull request


Instruction:
Here you have a function in Python 3, explain what it does
def __get_pull_review_comments(self, pr_number):

        comments = []
        group_comments = self.client.pull_review_comments(pr_number)

        for raw_comments in group_comments:

            for comment in json.loads(raw_comments):
                comment_id = comment.get('id')

                user = comment.get('user', None)
                if not user:
                    logger.warning("Missing user info for %s", comment['url'])
                    comment['user_data'] = None
                else:
                    comment['user_data'] = self.__get_user(user['login'])

                comment['reactions_data'] = \
                    self.__get_pull_review_comment_reactions(comment_id, comment['reactions']['total_count'])
                comments.append(comment)

        return comments

Response: Get all comments for a pull request


Instruction:
Implement a Python 3 function for
getting the list of reactions for a pull review comment

Response:
def __get_pull_review_comment_reactions(self, comment_id, total_count):
        """Get pull review comment reactions"""

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.pull_review_comment_reactions(comment_id)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions


Instruction:
Write a Python 3 script for
getting user and organizations data for the login

Response:
def __get_user(self, login):
        """Get user and org data for the login"""

        user = {}

        if not login:
            return user

        user_raw = self.client.user(login)
        user = json.loads(user_raw)
        user_orgs_raw = \
            self.client.user_orgs(login)
        user['organizations'] = json.loads(user_orgs_raw)

        return user


Instruction:
Make a summary of the following Python 3 code
def issue_reactions(self, issue_number):

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        path = urijoin("issues", str(issue_number), "reactions")
        return self.fetch_items(path, payload)

Response: Get reactions of an issue


Instruction:
Write a Python 3 function that can
fetch the issues from the GitHub repository.

Response:
def issues(self, from_date=None):
        """Fetch the issues from the repository.

        The method retrieves, from a GitHub repository, the issues
        updated since the given date.

        :param from_date: obtain issues updated since this date

        :returns: a generator of issues
        """
        payload = {
            'state': 'all',
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'}

        if from_date:
            payload['since'] = from_date.isoformat()

        path = urijoin("issues")
        return self.fetch_items(path, payload)


Instruction:
Create a Python 3 function to
fetch the pull requests from the GitHub repository.

Response:
def pulls(self, from_date=None):
        """Fetch the pull requests from the repository.

        The method retrieves, from a GitHub repository, the pull requests
        updated since the given date.

        :param from_date: obtain pull requests updated since this date

        :returns: a generator of pull requests
        """
        issues_groups = self.issues(from_date=from_date)

        for raw_issues in issues_groups:
            issues = json.loads(raw_issues)
            for issue in issues:

                if "pull_request" not in issue:
                    continue

                pull_number = issue["number"]
                path = urijoin(self.base_url, 'repos', self.owner, self.repository, "pulls", pull_number)

                r = self.fetch(path)
                pull = r.text

                yield pull


Instruction:
Can you tell what is the following Python 3 function doing
def repo(self):

        path = urijoin(self.base_url, 'repos', self.owner, self.repository)

        r = self.fetch(path)
        repo = r.text

        return repo

Response: Get the repository data


Instruction:
How would you explain what the following Python 3 function does
def pull_requested_reviewers(self, pr_number):

        requested_reviewers_url = urijoin("pulls", str(pr_number), "requested_reviewers")
        return self.fetch_items(requested_reviewers_url, {})

Response: Get a list of pull requested reviewers


Instruction:
Can you implement a function in Python 3 that
gets the commits for a pull request

Response:
def pull_commits(self, pr_number):
        """Get pull request commits"""

        payload = {
            'per_page': PER_PAGE,
        }

        commit_url = urijoin("pulls", str(pr_number), "commits")
        return self.fetch_items(commit_url, payload)


Instruction:
How would you explain what the following Python 3 function does
def pull_review_comments(self, pr_number):

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        comments_url = urijoin("pulls", str(pr_number), "comments")
        return self.fetch_items(comments_url, payload)

Response: Get the comments for a pull request


Instruction:
Given the following Python 3 function, write the documentation
def pull_review_comment_reactions(self, comment_id):

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        path = urijoin("pulls", "comments", str(comment_id), "reactions")
        return self.fetch_items(path, payload)

Response: Get reactions of a review comment


Instruction:
Write a Python 3 function for
getting the user information and update the user cache

Response:
def user(self, login):
        """Get the user information and update the user cache"""
        user = None

        if login in self._users:
            return self._users[login]

        url_user = urijoin(self.base_url, 'users', login)

        logging.info("Getting info for %s" % (url_user))

        r = self.fetch(url_user)
        user = r.text
        self._users[login] = user

        return user


Instruction:
Given the following Python 3 function, write the documentation
def user_orgs(self, login):
        if login in self._users_orgs:
            return self._users_orgs[login]

        url = urijoin(self.base_url, 'users', login, 'orgs')
        try:
            r = self.fetch(url)
            orgs = r.text
        except requests.exceptions.HTTPError as error:
            # 404 not found is wrongly received sometimes
            if error.response.status_code == 404:
                logger.error("Can't get github login orgs: %s", error)
                orgs = '[]'
            else:
                raise error

        self._users_orgs[login] = orgs

        return orgs

Response: Get the user public organizations


Instruction:
Given the following Python 3 function, write the documentation
def _get_token_rate_limit(self, token):

        rate_url = urijoin(self.base_url, "rate_limit")
        self.session.headers.update({'Authorization': 'token ' + token})
        remaining = 0
        try:
            headers = super().fetch(rate_url).headers
            if self.rate_limit_header in headers:
                remaining = int(headers[self.rate_limit_header])
        except requests.exceptions.HTTPError as error:
            logger.warning("Rate limit not initialized: %s", error)
        return remaining

Response: Return token s remaining API points


Instruction:
Write a Python 3 function for
returning array of all tokens remaining API points

Response:
def _get_tokens_rate_limits(self):
        """Return array of all tokens remaining API points"""

        remainings = [0] * self.n_tokens
        # Turn off archiving when checking rates, because that would cause
        # archive key conflict (the same URLs giving different responses)
        arch = self.archive
        self.archive = None
        for idx, token in enumerate(self.tokens):
            # Pass flag to skip disabling archiving because this function doies it
            remainings[idx] = self._get_token_rate_limit(token)
        # Restore archiving to whatever state it was
        self.archive = arch
        logger.debug("Remaining API points: {}".format(remainings))
        return remainings


Instruction:
Implement a Python 3 function for
checking all API tokens defined and choose one with most remaining API points and update the current token with the new one

Response:
def _choose_best_api_token(self):
        """Check all API tokens defined and choose one with most remaining API points"""

        # Return if no tokens given
        if self.n_tokens == 0:
            return

        # If multiple tokens given, choose best
        token_idx = 0
        if self.n_tokens > 1:
            remainings = self._get_tokens_rate_limits()
            token_idx = remainings.index(max(remainings))
            logger.debug("Remaining API points: {}, choosen index: {}".format(remainings, token_idx))

        # If we have any tokens - use best of them
        self.current_token = self.tokens[token_idx]
        self.session.headers.update({'Authorization': 'token ' + self.current_token})
        # Update rate limit data for the current token
        self._update_current_rate_limit()


Instruction:
Can you generate the documentation for the following Python 3 function
def _need_check_tokens(self):

        if self.n_tokens <= 1 or self.rate_limit is None:
            return False
        elif self.last_rate_limit_checked is None:
            self.last_rate_limit_checked = self.rate_limit
            return True

        # If approaching minimum rate limit for sleep
        approaching_limit = float(self.min_rate_to_sleep) * (1.0 + TOKEN_USAGE_BEFORE_SWITCH) + 1
        if self.rate_limit <= approaching_limit:
            self.last_rate_limit_checked = self.rate_limit
            return True

        # Only switch token when used predefined factor of the current token's remaining API points
        ratio = float(self.rate_limit) / float(self.last_rate_limit_checked)
        if ratio < 1.0 - TOKEN_USAGE_BEFORE_SWITCH:
            self.last_rate_limit_checked = self.rate_limit
            return True
        elif ratio > 1.0:
            self.last_rate_limit_checked = self.rate_limit
            return False
        else:
            return False

Response: Check if we need to switch GitHub API tokens


Instruction:
Implement a Python 3 function for
updating the rate limit data for the current token

Response:
def _update_current_rate_limit(self):
        """Update rate limits data for the current token"""

        url = urijoin(self.base_url, "rate_limit")
        try:
            # Turn off archiving when checking rates, because that would cause
            # archive key conflict (the same URLs giving different responses)
            arch = self.archive
            self.archive = None
            response = super().fetch(url)
            self.archive = arch
            self.update_rate_limit(response)
            self.last_rate_limit_checked = self.rate_limit
        except requests.exceptions.HTTPError as error:
            if error.response.status_code == 404:
                logger.warning("Rate limit not initialized: %s", error)
            else:
                raise error


Instruction:
How would you implement a function in Python 3 that
initializes the metadata of the items in the archive.

Response:
def init_metadata(self, origin, backend_name, backend_version,
                      category, backend_params):
        """Init metadata information.

        Metatada is composed by basic information needed to identify
        where archived data came from and how it can be retrieved
        and built into Perceval items.

        :param: origin: identifier of the repository
        :param: backend_name: name of the backend
        :param: backend_version: version of the backend
        :param: category: category of the items fetched
        :param: backend_params: dict representation of the fetch parameters

        raises ArchiveError: when an error occurs initializing the metadata
        """
        created_on = datetime_to_utc(datetime_utcnow())
        created_on_dumped = created_on.isoformat()
        backend_params_dumped = pickle.dumps(backend_params, 0)

        metadata = (origin, backend_name, backend_version, category,
                    backend_params_dumped, created_on_dumped,)

        try:
            cursor = self._db.cursor()
            insert_stmt = "INSERT INTO " + self.METADATA_TABLE + " "\
                          "(origin, backend_name, backend_version, " \
                          "category, backend_params, created_on) " \
                          "VALUES (?, ?, ?, ?, ?, ?)"
            cursor.execute(insert_stmt, metadata)

            self._db.commit()
            cursor.close()
        except sqlite3.DatabaseError as e:
            msg = "metadata initialization error; cause: %s" % str(e)
            raise ArchiveError(cause=msg)

        self.origin = origin
        self.backend_name = backend_name
        self.backend_version = backend_version
        self.category = category
        self.backend_params = backend_params
        self.created_on = created_on

        logger.debug("Metadata of archive %s initialized to %s",
                     self.archive_path, metadata)


Instruction:
Create a Python 3 function for
storing a raw item in this archive.

Response:
def store(self, uri, payload, headers, data):
        """Store a raw item in this archive.

        The method will store `data` content in this archive. The unique
        identifier for that item will be generated using the rest of the
        parameters.

        :param uri: request URI
        :param payload: request payload
        :param headers: request headers
        :param data: data to store in this archive

        :raises ArchiveError: when an error occurs storing the given data
        """
        hashcode = self.make_hashcode(uri, payload, headers)
        payload_dump = pickle.dumps(payload, 0)
        headers_dump = pickle.dumps(headers, 0)
        data_dump = pickle.dumps(data, 0)

        logger.debug("Archiving %s with %s %s %s in %s",
                     hashcode, uri, payload, headers, self.archive_path)

        try:
            cursor = self._db.cursor()
            insert_stmt = "INSERT INTO " + self.ARCHIVE_TABLE + " (" \
                          "id, hashcode, uri, payload, headers, data) " \
                          "VALUES(?,?,?,?,?,?)"
            cursor.execute(insert_stmt, (None, hashcode, uri,
                                         payload_dump, headers_dump, data_dump))
            self._db.commit()
            cursor.close()
        except sqlite3.IntegrityError as e:
            msg = "data storage error; cause: duplicated entry %s" % hashcode
            raise ArchiveError(cause=msg)
        except sqlite3.DatabaseError as e:
            msg = "data storage error; cause: %s" % str(e)
            raise ArchiveError(cause=msg)

        logger.debug("%s data archived in %s", hashcode, self.archive_path)


Instruction:
Write a Python 3 function that can
retrieve a raw item from the archive.

Response:
def retrieve(self, uri, payload, headers):
        """Retrieve a raw item from the archive.

        The method will return the `data` content corresponding to the
        hascode derived from the given parameters.

        :param uri: request URI
        :param payload: request payload
        :param headers: request headers

        :returns: the archived data

        :raises ArchiveError: when an error occurs retrieving data
        """
        hashcode = self.make_hashcode(uri, payload, headers)

        logger.debug("Retrieving entry %s with %s %s %s in %s",
                     hashcode, uri, payload, headers, self.archive_path)

        self._db.row_factory = sqlite3.Row

        try:
            cursor = self._db.cursor()
            select_stmt = "SELECT data " \
                          "FROM " + self.ARCHIVE_TABLE + " " \
                          "WHERE hashcode = ?"
            cursor.execute(select_stmt, (hashcode,))
            row = cursor.fetchone()
            cursor.close()
        except sqlite3.DatabaseError as e:
            msg = "data retrieval error; cause: %s" % str(e)
            raise ArchiveError(cause=msg)

        if row:
            found = pickle.loads(row['data'])
        else:
            msg = "entry %s not found in archive %s" % (hashcode, self.archive_path)
            raise ArchiveError(cause=msg)

        return found


Instruction:
Make a summary of the following Python 3 code
def create(cls, archive_path):
        if os.path.exists(archive_path):
            msg = "archive %s already exists; remove it before creating a new one"
            raise ArchiveError(cause=msg % (archive_path))

        conn = sqlite3.connect(archive_path)

        cursor = conn.cursor()
        cursor.execute(cls.METADATA_CREATE_STMT)
        cursor.execute(cls.ARCHIVE_CREATE_STMT)
        conn.commit()

        cursor.close()
        conn.close()

        logger.debug("Creating archive %s", archive_path)
        archive = cls(archive_path)
        logger.debug("Achive %s was created", archive_path)

        return archive

Response: Create a brand new archive.


Instruction:
Write a Python 3 function that can
generate a SHA1 hash code based on the given arguments.

Response:
def make_hashcode(uri, payload, headers):
        """Generate a SHA1 based on the given arguments.

        Hashcodes created by this method will used as unique identifiers
        for the raw items or resources stored by this archive.

        :param uri: URI to the resource
        :param payload: payload of the request needed to fetch the resource
        :param headers: headers of the request needed to fetch the resource

        :returns: a SHA1 hash code
        """
        def dict_to_json_str(data):
            return json.dumps(data, sort_keys=True)

        content = ':'.join([uri, dict_to_json_str(payload), dict_to_json_str(headers)])
        hashcode = hashlib.sha1(content.encode('utf-8'))
        return hashcode.hexdigest()


Instruction:
Write a Python 3 function for
checking whether the archive is valid or not. This method will check if tables were created and if they contain valid data.

Response:
def _verify_archive(self):
        """Check whether the archive is valid or not.

        This method will check if tables were created and if they
        contain valid data.
        """
        nentries = self._count_table_rows(self.ARCHIVE_TABLE)
        nmetadata = self._count_table_rows(self.METADATA_TABLE)

        if nmetadata > 1:
            msg = "archive %s metadata corrupted; multiple metadata entries" % (self.archive_path)
            raise ArchiveError(cause=msg)
        if nmetadata == 0 and nentries > 0:
            msg = "archive %s metadata is empty but %s entries were achived" % (self.archive_path)
            raise ArchiveError(cause=msg)

        logger.debug("Integrity of archive %s OK; entries: %s rows, metadata: %s rows",
                     self.archive_path, nentries, nmetadata)


Instruction:
Explain what the following Python 3 code does
def _load_metadata(self):

        logger.debug("Loading metadata infomation of archive %s", self.archive_path)

        cursor = self._db.cursor()
        select_stmt = "SELECT origin, backend_name, backend_version, " \
                      "category, backend_params, created_on " \
                      "FROM " + self.METADATA_TABLE + " " \
                      "LIMIT 1"
        cursor.execute(select_stmt)
        row = cursor.fetchone()
        cursor.close()

        if row:
            self.origin = row[0]
            self.backend_name = row[1]
            self.backend_version = row[2]
            self.category = row[3]
            self.backend_params = pickle.loads(row[4])
            self.created_on = str_to_datetime(row[5])
        else:
            logger.debug("Metadata of archive %s was empty", self.archive_path)

        logger.debug("Metadata of archive %s loaded", self.archive_path)

Response: Load the metadata from the archive file


Instruction:
Write a Python 3 script for
fetching the number of rows in a table

Response:
def _count_table_rows(self, table_name):
        """Fetch the number of rows in a table"""

        cursor = self._db.cursor()
        select_stmt = "SELECT COUNT(*) FROM " + table_name

        try:
            cursor.execute(select_stmt)
            row = cursor.fetchone()
        except sqlite3.DatabaseError as e:
            msg = "invalid archive file; cause: %s" % str(e)
            raise ArchiveError(cause=msg)
        finally:
            cursor.close()

        return row[0]


Instruction:
How would you implement a function in Python 3 that
creates a new archive with the given name and a random SHA1 hash.

Response:
def create_archive(self):
        """Create a new archive.

        The method creates in the filesystem a brand new archive with
        a random SHA1 as its name. The first byte of the hashcode will
        be the name of the subdirectory; the remaining bytes, the
        archive name.

        :returns: a new `Archive` object

        :raises ArchiveManagerError: when an error occurs creating the
            new archive
        """
        hashcode = uuid.uuid4().hex
        archive_dir = os.path.join(self.dirpath, hashcode[0:2])
        archive_name = hashcode[2:] + self.STORAGE_EXT
        archive_path = os.path.join(archive_dir, archive_name)

        if not os.path.exists(archive_dir):
            os.makedirs(archive_dir)

        try:
            archive = Archive.create(archive_path)
        except ArchiveError as e:
            raise ArchiveManagerError(cause=str(e))

        return archive