Instruction:
Write a Python 3 function that can
get any showcases the dataset is in

Response:
def get_showcases(self):
        # type: () -> List[hdx.data.showcase.Showcase]
        """Get any showcases the dataset is in

        Returns:
            List[Showcase]: list of showcases
        """
        assoc_result, showcases_dicts = self._read_from_hdx('showcase', self.data['id'], fieldname='package_id',
                                                            action=hdx.data.showcase.Showcase.actions()['list_showcases'])
        showcases = list()
        if assoc_result:
            for showcase_dict in showcases_dicts:
                showcase = hdx.data.showcase.Showcase(showcase_dict, configuration=self.configuration)
                showcases.append(showcase)
        return showcases


Instruction:
How would you explain what the following Python 3 function does
def _get_dataset_showcase_dict(self, showcase):
        # type: (Union[hdx.data.showcase.Showcase, Dict,str]) -> Dict
        """Get dataset showcase dict

        Args:
            showcase (Union[Showcase,Dict,str]): Either a showcase id or Showcase metadata from a Showcase object or dictionary

        Returns:
            dict: dataset showcase dict
        """
        if isinstance(showcase, hdx.data.showcase.Showcase) or isinstance(showcase, dict):
            if 'id' not in showcase:
                showcase = hdx.data.showcase.Showcase.read_from_hdx(showcase['name'])
            showcase = showcase['id']
        elif not isinstance(showcase, str):
            raise HDXError('Type %s cannot be added as a showcase!' % type(showcase).__name__)
        if is_valid_uuid(showcase) is False:
            raise HDXError('%s is not a valid showcase id!' % showcase)
        return {'package_id': self.data['id'], 'showcase_id': showcase}

Response: Get dataset showcase dict from a showcase object or a dictionary


Instruction:
Explain what the following Python 3 code does
def add_showcase(self, showcase, showcases_to_check=None):
        # type: (Union[hdx.data.showcase.Showcase,Dict,str], List[hdx.data.showcase.Showcase]) -> bool
        """Add dataset to showcase

        Args:
            showcase (Union[Showcase,Dict,str]): Either a showcase id or showcase metadata from a Showcase object or dictionary
            showcases_to_check (List[Showcase]): list of showcases against which to check existence of showcase. Defaults to showcases containing dataset.

        Returns:
            bool: True if the showcase was added, False if already present
        """
        dataset_showcase = self._get_dataset_showcase_dict(showcase)
        if showcases_to_check is None:
            showcases_to_check = self.get_showcases()
        for showcase in showcases_to_check:
            if dataset_showcase['showcase_id'] == showcase['id']:
                return False
        showcase = hdx.data.showcase.Showcase({'id': dataset_showcase['showcase_id']}, configuration=self.configuration)
        showcase._write_to_hdx('associate', dataset_showcase, 'package_id')
        return True

Response: Adds a dataset to a showcase.


Instruction:
Can you tell what is the following Python 3 function doing
def add_showcases(self, showcases, showcases_to_check=None):
        # type: (List[Union[hdx.data.showcase.Showcase,Dict,str]], List[hdx.data.showcase.Showcase]) -> bool
        """Add dataset to multiple showcases

        Args:
            showcases (List[Union[Showcase,Dict,str]]): A list of either showcase ids or showcase metadata from Showcase objects or dictionaries
            showcases_to_check (List[Showcase]): list of showcases against which to check existence of showcase. Defaults to showcases containing dataset.

        Returns:
            bool: True if all showcases added or False if any already present
        """
        if showcases_to_check is None:
            showcases_to_check = self.get_showcases()
        allshowcasesadded = True
        for showcase in showcases:
            if not self.add_showcase(showcase, showcases_to_check=showcases_to_check):
                allshowcasesadded = False
        return allshowcasesadded

Response: Add dataset to multiple showcases


Instruction:
Can you tell what is the following Python 3 function doing
def remove_showcase(self, showcase):
        # type: (Union[hdx.data.showcase.Showcase,Dict,str]) -> None
        """Remove dataset from showcase

        Args:
            showcase (Union[Showcase,Dict,str]): Either a showcase id string or showcase metadata from a Showcase object or dictionary

        Returns:
            None
        """
        dataset_showcase = self._get_dataset_showcase_dict(showcase)
        showcase = hdx.data.showcase.Showcase({'id': dataset_showcase['showcase_id']}, configuration=self.configuration)
        showcase._write_to_hdx('disassociate', dataset_showcase, 'package_id')

Response: Removes dataset from showcase


Instruction:
Write a Python 3 function that can
set the dataset to be of type requestable or notater.

Response:
def set_requestable(self, requestable=True):
        # type: (bool) -> None
        """Set the dataset to be of type requestable or not

        Args:
            requestable (bool): Set whether dataset is requestable. Defaults to True.

        Returns:
            None
        """
        self.data['is_requestdata_type'] = requestable
        if requestable:
            self.data['private'] = False


Instruction:
Make a summary of the following Python 3 code
def get_filetypes(self):
        # type: () -> List[str]
        """Return list of filetypes in your data

        Returns:
            List[str]: List of filetypes
        """
        if not self.is_requestable():
            return [resource.get_file_type() for resource in self.get_resources()]
        return self._get_stringlist_from_commastring('file_types')

Response: Return list of filetypes in your data


Instruction:
How would you implement a function in Python 3 that
cleans dataset tags according to tags cleanup spreadsheet and return if any changes occurred

Response:
def clean_dataset_tags(self):
        # type: () -> Tuple[bool, bool]
        """Clean dataset tags according to tags cleanup spreadsheet and return if any changes occurred

        Returns:
            Tuple[bool, bool]: Returns (True if tags changed or False if not, True if error or False if not)
        """
        tags_dict, wildcard_tags = Tags.tagscleanupdicts()

        def delete_tag(tag):
            logger.info('%s - Deleting tag %s!' % (self.data['name'], tag))
            return self.remove_tag(tag), False

        def update_tag(tag, final_tags, wording, remove_existing=True):
            text = '%s - %s: %s -> ' % (self.data['name'], wording, tag)
            if not final_tags:
                logger.error('%snothing!' % text)
                return False, True
            tags_lower_five = final_tags[:5].lower()
            if tags_lower_five == 'merge' or tags_lower_five == 'split' or (
                    ';' not in final_tags and len(final_tags) > 50):
                logger.error('%s%s - Invalid final tag!' % (text, final_tags))
                return False, True
            if remove_existing:
                self.remove_tag(tag)
            tags = ', '.join(self.get_tags())
            if self.add_tags(final_tags.split(';')):
                logger.info('%s%s! Dataset tags: %s' % (text, final_tags, tags))
            else:
                logger.warning(
                    '%s%s - At least one of the tags already exists! Dataset tags: %s' % (text, final_tags, tags))
            return True, False

        def do_action(tag, tags_dict_key):
            whattodo = tags_dict[tags_dict_key]
            action = whattodo[u'action']
            final_tags = whattodo[u'final tags (semicolon separated)']
            if action == u'Delete':
                changed, error = delete_tag(tag)
            elif action == u'Merge':
                changed, error = update_tag(tag, final_tags, 'Merging')
            elif action == u'Fix spelling':
                changed, error = update_tag(tag, final_tags, 'Fixing spelling')
            elif action == u'Non English':
                changed, error = update_tag(tag, final_tags, 'Anglicising', remove_existing=False)
            else:
                changed = False
                error = False
            return changed, error

        def process_tag(tag):
            changed = False
            error = False
            if tag in tags_dict.keys():
                changed, error = do_action(tag, tag)
            else:
                for wildcard_tag in wildcard_tags:
                    if fnmatch.fnmatch(tag, wildcard_tag):
                        changed, error = do_action(tag, wildcard_tag)
                        break
            return changed, error

        anychange = False
        anyerror = False
        for tag in self.get_tags():
            changed, error = process_tag(tag)
            if changed:
                anychange = True
            if error:
                anyerror = True

        return anychange, anyerror


Instruction:
How would you code a function in Python 3 to
set the resource that will be used for displaying QuickCharts in dataset preview

Response:
def set_quickchart_resource(self, resource):
        # type: (Union[hdx.data.resource.Resource,Dict,str,int]) -> bool
        """Set the resource that will be used for displaying QuickCharts in dataset preview

        Args:
            resource (Union[hdx.data.resource.Resource,Dict,str,int]): Either resource id or name, resource metadata from a Resource object or a dictionary or position

        Returns:
            bool: Returns True if resource for QuickCharts in dataset preview set or False if not
        """
        if isinstance(resource, int) and not isinstance(resource, bool):
            resource = self.get_resources()[resource]
        if isinstance(resource, hdx.data.resource.Resource) or isinstance(resource, dict):
            res = resource.get('id')
            if res is None:
                resource = resource['name']
            else:
                resource = res
        elif not isinstance(resource, str):
            raise hdx.data.hdxobject.HDXError('Resource id cannot be found in type %s!' % type(resource).__name__)
        if is_valid_uuid(resource) is True:
            search = 'id'
        else:
            search = 'name'
        changed = False
        for dataset_resource in self.resources:
            if dataset_resource[search] == resource:
                dataset_resource.enable_dataset_preview()
                self.preview_resource()
                changed = True
            else:
                dataset_resource.disable_dataset_preview()
        return changed


Instruction:
Implement a Python 3 function for
creating default resource views for all resources in dataset

Response:
def create_default_views(self, create_datastore_views=False):
        # type: (bool) -> None
        """Create default resource views for all resources in dataset

        Args:
            create_datastore_views (bool): Whether to try to create resource views that point to the datastore

        Returns:
            None
        """
        package = deepcopy(self.data)
        if self.resources:
            package['resources'] = self._convert_hdxobjects(self.resources)

        data = {'package': package, 'create_datastore_views': create_datastore_views}
        self._write_to_hdx('create_default_views', data, 'package')


Instruction:
Here you have a function in Python 3, explain what it does
def _get_credentials(self):
        # type: () -> Optional[Tuple[str, str]]
        """
        Return HDX site username and password

        Returns:
            Optional[Tuple[str, str]]: HDX site username and password or None

        """
        site = self.data[self.hdx_site]
        username = site.get('username')
        if username:
            return b64decode(username).decode('utf-8'), b64decode(site['password']).decode('utf-8')
        else:
            return None

Response: Returns username and password of the HDX site if username and password are not provided otherwise None.


Instruction:
Can you implement a function in Python 3 that
calls the remote CKAN method in the CKAN instance.

Response:
def call_remoteckan(self, *args, **kwargs):
        # type: (Any, Any) -> Dict
        """
        Calls the remote CKAN

        Args:
            *args: Arguments to pass to remote CKAN call_action method
            **kwargs: Keyword arguments to pass to remote CKAN call_action method

        Returns:
            Dict: The response from the remote CKAN call_action method

        """
        requests_kwargs = kwargs.get('requests_kwargs', dict())
        credentials = self._get_credentials()
        if credentials:
            requests_kwargs['auth'] = credentials
        kwargs['requests_kwargs'] = requests_kwargs
        apikey = kwargs.get('apikey', self.get_api_key())
        kwargs['apikey'] = apikey
        return self.remoteckan().call_action(*args, **kwargs)


Instruction:
Explain what the following Python 3 code does
def _environment_variables(**kwargs):
        # type: (Any) -> Any
        """
        Overwrite keyword arguments with environment variables

        Args:
            **kwargs: See below
            hdx_url (str): HDX url to use. Overrides hdx_site.
            hdx_site (str): HDX site to use eg. prod, test. Defaults to test.
            hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.

        Returns:
            kwargs: Changed keyword arguments

        """

        hdx_key = os.getenv('HDX_KEY')
        if hdx_key is not None:
            kwargs['hdx_key'] = hdx_key
        hdx_url = os.getenv('HDX_URL')
        if hdx_url is not None:
            kwargs['hdx_url'] = hdx_url
        else:
            hdx_site = os.getenv('HDX_SITE')
            if hdx_site is not None:
                kwargs['hdx_site'] = hdx_site
        return kwargs

Response: Returns a dictionary of keyword arguments with environment variables set in the object.


Instruction:
Given the following Python 3 function, write the documentation
def create_remoteckan(cls, site_url, user_agent=None, user_agent_config_yaml=None, user_agent_lookup=None,
                          session=None, **kwargs):
        # type: (str, Optional[str], Optional[str], Optional[str], requests.Session, Any) -> ckanapi.RemoteCKAN
        """
        Create remote CKAN instance from configuration

        Args:
            site_url (str): Site url.
            user_agent (Optional[str]): User agent string. HDXPythonLibrary/X.X.X- is prefixed.
            user_agent_config_yaml (Optional[str]): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
            user_agent_lookup (Optional[str]): Lookup key for YAML. Ignored if user_agent supplied.
            session (requests.Session): requests Session object to use. Defaults to calling hdx.utilities.session.get_session()

        Returns:
            ckanapi.RemoteCKAN: Remote CKAN instance

        """
        if not session:
            session = get_session(user_agent, user_agent_config_yaml, user_agent_lookup, prefix=Configuration.prefix,
                                  method_whitelist=frozenset(['HEAD', 'TRACE', 'GET', 'POST', 'PUT',
                                                              'OPTIONS', 'DELETE']), **kwargs)
            ua = session.headers['User-Agent']
        else:
            ua = kwargs.get('full_agent')
            if not ua:
                ua = UserAgent.get(user_agent, user_agent_config_yaml, user_agent_lookup, prefix=Configuration.prefix,
                                   **kwargs)
        return ckanapi.RemoteCKAN(site_url, user_agent=ua, session=session)

Response: Create CKAN instance from configuration.


Instruction:
Can you write a function in Python 3 where it
sets up remote CKAN from provided CKAN or by creating from configuration Args: remoteckan (Optional[ckanapi.RemoteCKAN]): CKAN instance. Defaults to setting one up from configuration. Returns: None

Response:
def setup_remoteckan(self, remoteckan=None, **kwargs):
        # type: (Optional[ckanapi.RemoteCKAN], Any) -> None
        """
        Set up remote CKAN from provided CKAN or by creating from configuration

        Args:
            remoteckan (Optional[ckanapi.RemoteCKAN]): CKAN instance. Defaults to setting one up from configuration.

        Returns:
            None

        """
        if remoteckan is None:
            self._remoteckan = self.create_remoteckan(self.get_hdx_site_url(), full_agent=self.get_user_agent(),
                                                      **kwargs)
        else:
            self._remoteckan = remoteckan


Instruction:
Can you create a Python 3 function that
sets up the HDX configuration object.

Response:
def setup(cls, configuration=None, **kwargs):
        # type: (Optional['Configuration'], Any) -> None
        """
        Set up the HDX configuration

        Args:
            configuration (Optional[Configuration]): Configuration instance. Defaults to setting one up from passed arguments.
            **kwargs: See below
            user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not.
            user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
            user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
            hdx_url (str): HDX url to use. Overrides hdx_site.
            hdx_site (str): HDX site to use eg. prod, test.
            hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
            hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
            hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
            hdx_config_json (str): Path to JSON HDX configuration OR
            hdx_config_yaml (str): Path to YAML HDX configuration
            project_config_dict (dict): Project configuration dictionary OR
            project_config_json (str): Path to JSON Project configuration OR
            project_config_yaml (str): Path to YAML Project configuration
            hdx_base_config_dict (dict): HDX base configuration dictionary OR
            hdx_base_config_json (str): Path to JSON HDX base configuration OR
            hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml.

        Returns:
            None

        """
        if configuration is None:
            cls._configuration = Configuration(**kwargs)
        else:
            cls._configuration = configuration


Instruction:
Implement a Python 3 function for
creating HDX configuration object.

Response:
def _create(cls, configuration=None, remoteckan=None, **kwargs):
        # type: (Optional['Configuration'], Optional[ckanapi.RemoteCKAN], Any) -> str
        """
        Create HDX configuration

        Args:
            configuration (Optional[Configuration]): Configuration instance. Defaults to setting one up from passed arguments.
            remoteckan (Optional[ckanapi.RemoteCKAN]): CKAN instance. Defaults to setting one up from configuration.
            **kwargs: See below
            user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not.
            user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
            user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
            hdx_url (str): HDX url to use. Overrides hdx_site.
            hdx_site (str): HDX site to use eg. prod, test.
            hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
            hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
            hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
            hdx_config_json (str): Path to JSON HDX configuration OR
            hdx_config_yaml (str): Path to YAML HDX configuration
            project_config_dict (dict): Project configuration dictionary OR
            project_config_json (str): Path to JSON Project configuration OR
            project_config_yaml (str): Path to YAML Project configuration
            hdx_base_config_dict (dict): HDX base configuration dictionary OR
            hdx_base_config_json (str): Path to JSON HDX base configuration OR
            hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml.

        Returns:
            str: HDX site url

        """
        kwargs = cls._environment_variables(**kwargs)
        cls.setup(configuration, **kwargs)
        cls._configuration.setup_remoteckan(remoteckan, **kwargs)
        return cls._configuration.get_hdx_site_url()


Instruction:
Can you tell what is the following Python 3 function doing
def create(cls, configuration=None, remoteckan=None, **kwargs):
        # type: (Optional['Configuration'], Optional[ckanapi.RemoteCKAN], Any) -> str
        """
        Create HDX configuration. Can only be called once (will raise an error if called more than once).

        Args:
            configuration (Optional[Configuration]): Configuration instance. Defaults to setting one up from passed arguments.
            remoteckan (Optional[ckanapi.RemoteCKAN]): CKAN instance. Defaults to setting one up from configuration.
            **kwargs: See below
            user_agent (str): User agent string. HDXPythonLibrary/X.X.X- is prefixed. Must be supplied if remoteckan is not.
            user_agent_config_yaml (str): Path to YAML user agent configuration. Ignored if user_agent supplied. Defaults to ~/.useragent.yml.
            user_agent_lookup (str): Lookup key for YAML. Ignored if user_agent supplied.
            hdx_url (str): HDX url to use. Overrides hdx_site.
            hdx_site (str): HDX site to use eg. prod, test.
            hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
            hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
            hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
            hdx_config_json (str): Path to JSON HDX configuration OR
            hdx_config_yaml (str): Path to YAML HDX configuration
            project_config_dict (dict): Project configuration dictionary OR
            project_config_json (str): Path to JSON Project configuration OR
            project_config_yaml (str): Path to YAML Project configuration
            hdx_base_config_dict (dict): HDX base configuration dictionary OR
            hdx_base_config_json (str): Path to JSON HDX base configuration OR
            hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yml.

        Returns:
            str: HDX site url

        """
        if cls._configuration is not None:
            raise ConfigurationError('Configuration already created!')
        return cls._create(configuration=configuration, remoteckan=remoteckan, **kwargs)

Response: Create HDX configuration object.


Instruction:
Implement a function in Python 3 to
convert a dictionary into a string with assignments

Response:
def kwargs_to_variable_assignment(kwargs: dict, value_representation=repr,
                                  assignment_operator: str = ' = ',
                                  statement_separator: str = '\n',
                                  statement_per_line: bool = False) -> str:
    """
    Convert a dictionary into a string with assignments

    Each assignment is constructed based on:
    key assignment_operator value_representation(value) statement_separator,
    where key and value are the key and value of the dictionary.
    Moreover one can seprate the assignment statements by new lines.

    Parameters
    ----------
    kwargs : dict

    assignment_operator: str, optional:
        Assignment operator (" = " in python)
    value_representation: str, optinal
        How to represent the value in the assignments (repr function in python)
    statement_separator : str, optional:
        Statement separator (new line in python)
    statement_per_line: bool, optional
        Insert each statement on a different line

    Returns
    -------
    str
        All the assignemnts.

    >>> kwargs_to_variable_assignment({'a': 2, 'b': "abc"})
    "a = 2\\nb = 'abc'\\n"
    >>> kwargs_to_variable_assignment({'a':2 ,'b': "abc"}, statement_per_line=True)
    "a = 2\\n\\nb = 'abc'\\n"
    >>> kwargs_to_variable_assignment({'a': 2})
    'a = 2\\n'
    >>> kwargs_to_variable_assignment({'a': 2}, statement_per_line=True)
    'a = 2\\n'
    """
    code = []
    join_str = '\n' if statement_per_line else ''
    for key, value in kwargs.items():
        code.append(key + assignment_operator +
                    value_representation(value)+statement_separator)
    return join_str.join(code)


Instruction:
Here you have a function in Python 3, explain what it does
def decode_json(json_input: Union[str, None] = None):
    if json_input is None:
        return {}
    else:
        if isinstance(json_input, str) is False:
            raise TypeError()
        elif json_input[-5:] == ".json":
            with open(json_input) as f:
                decoded_json = json.load(f)
        else:
            decoded_json = json.loads(json_input)
    return decoded_json

Response: Simple wrapper of json. load and json. loads.


Instruction:
Write a Python 3 script to
check if an object is json serialisable.

Response:
def is_jsonable(obj) -> bool:
    """
    Check if an object is jsonable.

    An object is jsonable if it is json serialisable and by loading its json representation the same object is recovered.
    Parameters
    ----------
    obj :
        Python object

    Returns
    -------
    bool

    >>> is_jsonable([1,2,3])
    True
    >>> is_jsonable((1,2,3))
    False
    >>> is_jsonable({'a':True,'b':1,'c':None})
    True
    """
    try:
        return obj==json.loads(json.dumps(obj))
    except TypeError:
        return False
    except:
        raise


Instruction:
Can you create a Python 3 function that
checks if an expresion can be literal_eval.

Response:
def is_literal_eval(node_or_string) -> tuple:
    """
    Check if an expresion can be literal_eval.

    ----------
    node_or_string :
        Input

    Returns
    -------
    tuple
        (bool,python object)

        If it can be literal_eval the python object is returned. Otherwise None it is returned.

    >>> is_literal_eval('[1,2,3]')
    (True, [1, 2, 3])
    >>> is_literal_eval('a')
    (False, None)
    """
    try:
        obj=ast.literal_eval(node_or_string)
        return (True, obj)
    except:
        return (False, None)


Instruction:
Write a Python 3 script for
returning the duplicates in a list.

Response:
def find_duplicates(l: list) -> set:
    """
    Return the duplicates in a list.

    The function relies on
    https://stackoverflow.com/questions/9835762/find-and-list-duplicates-in-a-list .
    Parameters
    ----------
    l : list
        Name

    Returns
    -------
    set
        Duplicated values

    >>> find_duplicates([1,2,3])
    set()
    >>> find_duplicates([1,2,1])
    {1}
    """
    return set([x for x in l if l.count(x) > 1])


Instruction:
Explain what the following Python 3 code does
def sort_dict(d: dict, by: str = 'key',
              allow_duplicates: bool = True) -> collections.OrderedDict:
    """
    Sort a dictionary by key or value.

    The function relies on
    https://docs.python.org/3/library/collections.html#collections.OrderedDict .
    The dulicated are determined based on
    https://stackoverflow.com/questions/9835762/find-and-list-duplicates-in-a-list .
    Parameters
    ----------
    d : dict
        Input dictionary
    by : ['key','value'], optional
        By what to sort the input dictionary
    allow_duplicates : bool, optional
        Flag to indicate if the duplicates are allowed.
    Returns
    -------
    collections.OrderedDict
        Sorted dictionary.

    >>> sort_dict({2: 3, 1: 2, 3: 1})
    OrderedDict([(1, 2), (2, 3), (3, 1)])
    >>> sort_dict({2: 3, 1: 2, 3: 1}, by='value')
    OrderedDict([(3, 1), (1, 2), (2, 3)])
    >>> sort_dict({'2': 3, '1': 2}, by='value')
    OrderedDict([('1', 2), ('2', 3)])
    >>> sort_dict({2: 1, 1: 2, 3: 1}, by='value', allow_duplicates=False)
    Traceback (most recent call last):
        ...
    ValueError: There are duplicates in the values: {1}
    >>> sort_dict({1:1,2:3},by=True)
    Traceback (most recent call last):
        ...
    ValueError: by can be 'key' or 'value'.
    """
    if by == 'key':
        i = 0
    elif by == 'value':
        values = list(d.values())
        if len(values) != len(set(values)) and not allow_duplicates:
            duplicates = find_duplicates(values)
            raise ValueError("There are duplicates in the values: {}".format(duplicates))
        i = 1
    else:
        raise ValueError("by can be 'key' or 'value'.")

    return collections.OrderedDict(sorted(d.items(), key=lambda t: t[i]))

Response: Sort a dictionary by key or value.


Instruction:
Write a Python 3 function for
grouping a dictionary by values.

Response:
def group_dict_by_value(d: dict) -> dict:
    """
    Group a dictionary by values.

    Parameters
    ----------
    d : dict
        Input dictionary

    Returns
    -------
    dict
        Output dictionary. The keys are the values of the initial dictionary
        and the values ae given by a list of keys corresponding to the value.

    >>> group_dict_by_value({2: 3, 1: 2, 3: 1})
    {3: [2], 2: [1], 1: [3]}
    >>> group_dict_by_value({2: 3, 1: 2, 3: 1, 10:1, 12: 3})
    {3: [2, 12], 2: [1], 1: [3, 10]}
    """
    d_out = {}
    for k, v in d.items():
        if v in d_out:
            d_out[v].append(k)
        else:
            d_out[v] = [k]
    return d_out


Instruction:
Here you have a function in Python 3, explain what it does
def variable_status(code: str,
                    exclude_variable: Union[set, None] = None,
                    jsonable_parameter: bool = True) -> tuple:
    """
    Find the possible parameters and "global" variables from a python code.

    This is achieved by parsing the abstract syntax tree.

    Parameters
    ----------
    code : str
        Input code as string.
    exclude_variable : set, None, optional
        Variable to exclude.
    jsonable_parameter: bool, True, optional
        Consider only jsonable parameter

    Returns
    -------
    tuple
        (a set of possible parameter, a set of parameter to exclude, a dictionary of possible parameter )

        A variable is a possible parameter if 1) it is not in the input exclude_variable,
        2) the code contains only assignments, and 3) it is used only to bound objects.

        The set of parameter to exclude is the union of the input exclude_variable and all names that looks like a global variable.
        The dictionary of possible parameter {parameter name, parameter value} is available only if jsonable_parameter is True.

    >>> variable_status("a=3")
    ({'a'}, {'a'}, {'a': 3})
    >>> variable_status("a=3",jsonable_parameter=False)
    ({'a'}, {'a'}, {})
    >>> variable_status("a += 1")
    (set(), {'a'}, {})
    >>> variable_status("def f(x,y=3):\\n\\t pass")
    (set(), {'f'}, {})
    >>> variable_status("class C(A):\\n\\t pass")
    (set(), {'C'}, {})
    >>> variable_status("import f")
    (set(), {'f'}, {})
    >>> variable_status("import f as g")
    (set(), {'g'}, {})
    >>> variable_status("from X import f")
    (set(), {'f'}, {})
    >>> variable_status("from X import f as g")
    (set(), {'g'}, {})
    """
    if exclude_variable is None:
        exclude_variable = set()
    else:
        exclude_variable = copy.deepcopy(exclude_variable)
    root = ast.parse(code)
    store_variable_name = set()
    assign_only = True
    dict_parameter={}

    for node in ast.iter_child_nodes(root):
        if isinstance(node, ast.Assign):
            for assign_node in ast.walk(node):
                if isinstance(assign_node, ast.Name):

                    if isinstance(assign_node.ctx, ast.Store):
                        if jsonable_parameter is False:
                            store_variable_name |= {assign_node.id}
                    else:
                        exclude_variable |= {assign_node.id}

            _is_literal_eval,_value=is_literal_eval(node.value)

            if jsonable_parameter is True:
                for assign_node in ast.iter_child_nodes(node):
                    if isinstance(assign_node, ast.Tuple):
                        i=0
                        for assign_tuple_node in ast.iter_child_nodes(assign_node):
                            if isinstance(assign_tuple_node, ast.Name):

                                if isinstance(_value,(collections.Iterable)) and is_jsonable(_value[i]) and _is_literal_eval:
                                    dict_parameter[assign_tuple_node.id]=_value[i]
                                    store_variable_name |= {assign_tuple_node.id}
                                else:
                                    exclude_variable |= {assign_tuple_node.id}
                                i += 1
                    else:
                        if isinstance(assign_node, ast.Name):
                            if is_jsonable(_value) and _is_literal_eval:
                                dict_parameter[assign_node.id]=_value
                                store_variable_name |= {assign_node.id}
                            else:
                                exclude_variable |= {assign_node.id}

        elif isinstance(node, ast.AugAssign):
            for assign_node in ast.walk(node):
                if isinstance(assign_node, ast.Name):
                    exclude_variable |= {assign_node.id}
        # class and function
        elif isinstance(node, (ast.FunctionDef, ast.ClassDef)):
            assign_only = False
            exclude_variable |= {node.name}
        # import
        elif isinstance(node, ast.Import):
            assign_only = False
            for node1 in ast.iter_child_nodes(node):
                if node1.asname is not None:
                    exclude_variable |= {node1.asname}
                else:
                    exclude_variable |= {node1.name}
        # import from
        elif isinstance(node, ast.ImportFrom):
            assign_only = False
            for node1 in ast.iter_child_nodes(node):
                if node1.asname is not None:
                    exclude_variable |= {node1.asname}
                else:
                    exclude_variable |= {node1.name}
        else:
            assign_only = False
    if assign_only is True:
        possible_parameter = store_variable_name-exclude_variable
        if jsonable_parameter is True:
            dict_parameter = {k:dict_parameter[k] for k in possible_parameter}
        return (possible_parameter, store_variable_name | exclude_variable, dict_parameter)
    return set(), store_variable_name | exclude_variable, {}

Response: Return the list of possible parameters and global variables from a python code.


Instruction:
How would you implement a function in Python 3 that
increments the name where the incremental part is given by parameters.

Response:
def increment_name(name: str, start_marker: str = " (",
                   end_marker: str = ")") -> str:
    """
    Increment the name where the incremental part is given by parameters.

    Parameters
    ----------
    name : str, nbformat.notebooknode.NotebookNode
        Name
    start_marker : str
        The marker used before the incremental
    end_marker : str
        The marker after the incrementa

    Returns
    -------
    str
        Incremented name.

    >>> increment_name('abc')
    'abc (1)'
    >>> increment_name('abc(1)')
    'abc(1) (1)'
    >>> increment_name('abc (123)')
    'abc (124)'
    >>> increment_name('abc-1',start_marker='-',end_marker='')
    'abc-2'
    >>> increment_name('abc[2]',start_marker='[',end_marker=']')
    'abc[3]'
    >>> increment_name('abc1',start_marker='',end_marker='')
    Traceback (most recent call last):
        ...
    ValueError: start_marker can not be the empty string.
    """
    if start_marker == '':
        raise ValueError("start_marker can not be the empty string.")
    a = name
    start = len(a)-a[::-1].find(start_marker[::-1])

    if (a[len(a)-len(end_marker):len(a)] == end_marker
            and start < (len(a)-len(end_marker))
            and a[start-len(start_marker):start] == start_marker
            and a[start:len(a)-len(end_marker)].isdigit()):

        old_int = int(a[start:len(a)-len(end_marker)])
        new_int = old_int+1
        new_name = a[:start]+str(new_int)+end_marker
    else:
        new_name = a+start_marker+'1'+end_marker
    return new_name


Instruction:
Given the following Python 3 function, write the documentation
def read_from_hdx(identifier, configuration=None):
        # type: (str, Optional[Configuration]) -> Optional['ResourceView']
        """Reads the resource view given by identifier from HDX and returns ResourceView object

        Args:
            identifier (str): Identifier of resource view
            configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration.

        Returns:
            Optional[ResourceView]: ResourceView object if successful read, None if not
        """

        resourceview = ResourceView(configuration=configuration)
        result = resourceview._load_from_hdx('resource view', identifier)
        if result:
            return resourceview
        return None

Response: Reads the resource view given by identifier from HDX and returns ResourceView object if successful.


Instruction:
Explain what the following Python 3 code does
def get_all_for_resource(identifier, configuration=None):
        # type: (str, Optional[Configuration]) -> List['ResourceView']
        """Read all resource views for a resource given by identifier from HDX and returns list of ResourceView objects

        Args:
            identifier (str): Identifier of resource
            configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration.

        Returns:
            List[ResourceView]: List of ResourceView objects
        """

        resourceview = ResourceView(configuration=configuration)
        success, result = resourceview._read_from_hdx('resource view', identifier, 'id', ResourceView.actions()['list'])
        resourceviews = list()
        if success:
            for resourceviewdict in result:
                resourceview = ResourceView(resourceviewdict, configuration=configuration)
                resourceviews.append(resourceview)
        return resourceviews

Response: Read all resource views for a resource given by identifier from HDX and return list of ResourceView objects.


Instruction:
Write a Python 3 script to
check if resource view exists in HDX and if so update resource view

Response:
def _update_resource_view(self, log=False):
        # type: () -> bool
        """Check if resource view exists in HDX and if so, update resource view

        Returns:
            bool: True if updated and False if not
        """
        update = False
        if 'id' in self.data and self._load_from_hdx('resource view', self.data['id']):
            update = True
        else:
            if 'resource_id' in self.data:
                resource_views = self.get_all_for_resource(self.data['resource_id'])
                for resource_view in resource_views:
                    if self.data['title'] == resource_view['title']:
                        self.old_data = self.data
                        self.data = resource_view.data
                        update = True
                        break
        if update:
            if log:
                logger.warning('resource view exists. Updating %s' % self.data['id'])
            self._merge_hdx_update('resource view', 'id')
        return update


Instruction:
Can you generate the documentation for the following Python 3 function
def create_in_hdx(self):
        # type: () -> None
        """Check if resource view exists in HDX and if so, update it, otherwise create resource view

        Returns:
            None
        """
        self.check_required_fields()
        if not self._update_resource_view(log=True):
            self._save_to_hdx('create', 'title')

Response: Check if resource view exists in HDX and if so create it otherwise create resource view


Instruction:
Can you write a function in Python 3 where it
copies all fields except id resource_id and package_id from another resource view.

Response:
def copy(self, resource_view):
        # type: (Union[ResourceView,Dict,str]) -> None
        """Copies all fields except id, resource_id and package_id from another resource view.

        Args:
            resource_view (Union[ResourceView,Dict,str]): Either a resource view id or resource view metadata either from a ResourceView object or a dictionary

        Returns:
            None
        """
        if isinstance(resource_view, str):
            if is_valid_uuid(resource_view) is False:
                raise HDXError('%s is not a valid resource view id!' % resource_view)
            resource_view = ResourceView.read_from_hdx(resource_view)
        if not isinstance(resource_view, dict) and not isinstance(resource_view, ResourceView):
            raise HDXError('%s is not a valid resource view!' % resource_view)
        for key in resource_view:
            if key not in ('id', 'resource_id', 'package_id'):
                self.data[key] = resource_view[key]


Instruction:
Can you generate the documentation for the following Python 3 function
def tagscleanupdicts(configuration=None, url=None, keycolumn=5, failchained=True):
        # type: (Optional[Configuration], Optional[str], int, bool) -> Tuple[Dict,List]
        """
        Get tags cleanup dictionaries

        Args:
            configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration.
            url (Optional[str]): Url of tags cleanup spreadsheet. Defaults to None (internal configuration parameter).
            keycolumn (int): Column number of tag column in spreadsheet. Defaults to 5.
            failchained (bool): Fail if chained rules found. Defaults to True.

        Returns:
            Tuple[Dict,List]: Returns (Tags dictionary, Wildcard tags list)
        """
        if not Tags._tags_dict:
            if configuration is None:
                configuration = Configuration.read()
            with Download(full_agent=configuration.get_user_agent()) as downloader:
                if url is None:
                    url = configuration['tags_cleanup_url']
                Tags._tags_dict = downloader.download_tabular_rows_as_dicts(url, keycolumn=keycolumn)
                keys = Tags._tags_dict.keys()
                chainerror = False
                for i, tag in enumerate(keys):
                    whattodo = Tags._tags_dict[tag]
                    action = whattodo[u'action']
                    final_tags = whattodo[u'final tags (semicolon separated)']
                    for final_tag in final_tags.split(';'):
                        if final_tag in keys:
                            index = list(keys).index(final_tag)
                            if index != i:
                                whattodo2 = Tags._tags_dict[final_tag]
                                action2 = whattodo2[u'action']
                                if action2 != 'OK' and action2 != 'Other':
                                    final_tags2 = whattodo2[u'final tags (semicolon separated)']
                                    if final_tag not in final_tags2.split(';'):
                                        chainerror = True
                                        if failchained:
                                            logger.error('Chained rules: %s (%s -> %s) | %s (%s -> %s)' %
                                                         (action, tag, final_tags, action2, final_tag, final_tags2))

                if failchained and chainerror:
                    raise ChainRuleError('Chained rules for tags detected!')
                Tags._wildcard_tags = list()
                for tag in Tags._tags_dict:
                    if '*' in tag:
                        Tags._wildcard_tags.append(tag)
        return Tags._tags_dict, Tags._wildcard_tags

Response: Get tags cleanup dictionaries.


Instruction:
Given the following Python 3 function, write the documentation
def read_from_hdx(identifier, configuration=None):
        # type: (str, Optional[Configuration]) -> Optional['User']
        """Reads the user given by identifier from HDX and returns User object

        Args:
            identifier (str): Identifier of user
            configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration.

        Returns:
            Optional[User]: User object if successful read, None if not
        """

        user = User(configuration=configuration)
        result = user._load_from_hdx('user', identifier)
        if result:
            return user
        return None

Response: Reads the user given by identifier from HDX and returns User object if successful.


Instruction:
How would you explain what the following Python 3 function does
def update_in_hdx(self):
        # type: () -> None
        """Check if user exists in HDX and if so, update user

        Returns:
            None
        """
        capacity = self.data.get('capacity')
        if capacity is not None:
            del self.data['capacity']  # remove capacity (which comes from users from Organization)
        self._update_in_hdx('user', 'id')
        if capacity is not None:
            self.data['capacity'] = capacity

Response: Check if user exists in HDX and if so update user


Instruction:
How would you implement a function in Python 3 that
checks if user exists in HDX and if so create it otherwise create user

Response:
def create_in_hdx(self):
        # type: () -> None
        """Check if user exists in HDX and if so, update it, otherwise create user

        Returns:
            None
        """
        capacity = self.data.get('capacity')
        if capacity is not None:
            del self.data['capacity']
        self._create_in_hdx('user', 'id', 'name')
        if capacity is not None:
            self.data['capacity'] = capacity


Instruction:
How would you explain what the following Python 3 function does
def email(self, subject, text_body, html_body=None, sender=None, **kwargs):
        # type: (str, str, Optional[str], Optional[str], Any) -> None
        """Emails a user.

        Args:
            subject (str): Email subject
            text_body (str): Plain text email body
            html_body (str): HTML email body
            sender (Optional[str]): Email sender. Defaults to SMTP username.
            **kwargs: See below
            mail_options (List): Mail options (see smtplib documentation)
            rcpt_options (List): Recipient options (see smtplib documentation)

        Returns:
            None
        """
        self.configuration.emailer().send([self.data['email']], subject, text_body, html_body=html_body, sender=sender,
                                          **kwargs)

Response:
Emails a user.

        Args:
            subject (str): Email subject
            text_body (str): Plain text email body
            html_body (str): HTML email body
            sender (Optional[str]): Email sender. Defaults to SMTP username.
            **kwargs: See below
            mail_options (List): Mail options (see smtplib documentation)
            rcpt_options (List): Recipient options (see smtplib documentation)

        Returns:
            None


Instruction:
Can you create a Python 3 function that
gets all users in HDX

Response:
def get_all_users(configuration=None, **kwargs):
        # type: (Optional[Configuration], Any) -> List['User']
        """Get all users in HDX

        Args:
            configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration.
            **kwargs: See below
            q (str): Restrict to names containing a string. Defaults to all users.
            order_by (str): Field by which to sort - any user field or edits (number_of_edits). Defaults to 'name'.

        Returns:
            List[User]: List of all users in HDX
        """
        user = User(configuration=configuration)
        user['id'] = 'all users'  # only for error message if produced
        result = user._write_to_hdx('list', kwargs, 'id')
        users = list()
        if result:
            for userdict in result:
                user = User(userdict, configuration=configuration)
                users.append(user)
        else:
            logger.debug(result)
        return users


Instruction:
Can you generate a brief explanation for the following Python 3 code
def email_users(users, subject, text_body, html_body=None, sender=None, configuration=None, **kwargs):
        # type: (List['User'], str, str, Optional[str], Optional[str], Optional[Configuration], Any) -> None
        """Email a list of users

        Args:
            users (List[User]): List of users
            subject (str): Email subject
            text_body (str): Plain text email body
            html_body (str): HTML email body
            sender (Optional[str]): Email sender. Defaults to SMTP username.
            configuration (Optional[Configuration]): HDX configuration. Defaults to configuration of first user in list.
            **kwargs: See below
            mail_options (List): Mail options (see smtplib documentation)
            rcpt_options (List): Recipient options (see smtplib documentation)

        Returns:
            None
        """
        if not users:
            raise ValueError('No users supplied')
        recipients = list()
        for user in users:
            recipients.append(user.data['email'])
        if configuration is None:
            configuration = users[0].configuration
        configuration.emailer().send(recipients, subject, text_body, html_body=html_body, sender=sender, **kwargs)

Response: Email a list of users to the HDX server.


Instruction:
Make a summary of the following Python 3 code
def get_organizations(self, permission='read'):
        # type: (str) -> List['Organization']
        """Get organizations in HDX that this user is a member of.

        Args:
            permission (str): Permission to check for. Defaults to 'read'.

        Returns:
            List[Organization]: List of organizations in HDX that this user is a member of
        """
        success, result = self._read_from_hdx('user', self.data['name'], 'id', self.actions()['listorgs'],
                                              permission=permission)
        organizations = list()
        if success:
            for organizationdict in result:
                organization = hdx.data.organization.Organization.read_from_hdx(organizationdict['id'])
                organizations.append(organization)
        return organizations

Response: Get organizations in HDX that this user is a member of.


Instruction:
Write a Python 3 script for
trying loading the config from the given path. If no path is specified the default config is returned.

Response:
def get_lint_config(config_path=None):
    """ Tries loading the config from the given path. If no path is specified, the default config path
    is tried, and if that is not specified, we the default config is returned. """
    # config path specified
    if config_path:
        config = LintConfig.load_from_file(config_path)
        click.echo("Using config from {0}".format(config_path))
    # default config path
    elif os.path.exists(DEFAULT_CONFIG_FILE):
        config = LintConfig.load_from_file(DEFAULT_CONFIG_FILE)
        click.echo("Using config from {0}".format(DEFAULT_CONFIG_FILE))
    # no config file
    else:
        config = LintConfig()

    return config


Instruction:
Make a summary of the following Python 3 code
def main():
    # Read configuration from the config file if present, else fall back to
    # command line options
    if args.config:
        config = config_file_parser.get_configuration(args.config)
        access_key_id = config['access-key-id']
        secret_access_key = config['secret-access-key']
        region = config['region']
    else:
        access_key_id = args.access_key_id
        secret_access_key = args.secret_access_key
        region = args.region

    if args.daemon:
        pid_file = '/tmp/automatic-ebs-snapshots.pid'
        daemon = AutoEBSDaemon(pid_file)

        if args.daemon == 'start':
            daemon.start()

        elif args.daemon == 'stop':
            daemon.stop()
            sys.exit(0)

        elif args.daemon == 'restart':
            daemon.restart()

        elif args.daemon in ['foreground', 'fg']:
            daemon.run()

        else:
            print 'Valid options for --daemon are start, stop and restart'
            sys.exit(1)

    # Connect to AWS
    connection = connection_manager.connect_to_ec2(
        region, access_key_id, secret_access_key)

    if args.watch:
        volume_manager.watch(
            connection,
            args.watch,
            args.interval,
            args.retention)

    if args.unwatch:
        volume_manager.unwatch(connection, args.unwatch)

    if args.watch_file:
        volume_manager.watch_from_file(connection, args.watch_file)

    if args.unwatch_file:
        volume_manager.unwatch_from_file(connection, args.unwatch_file)

    if args.snapshots:
        volume_manager.list_snapshots(connection, args.snapshots)

    if args.list:
        volume_manager.list(connection)

    if args.run:
        snapshot_manager.run(connection)

Response: Main function for the main function of the main function.


Instruction:
Can you tell what is the following Python 3 function doing
def run(self, check_interval=300):
        while True:
            # Read configuration from the config file if present, else fall
            # back to command line options
            if args.config:
                config = config_file_parser.get_configuration(args.config)
                access_key_id = config['access-key-id']
                secret_access_key = config['secret-access-key']
                region = config['region']
            else:
                access_key_id = args.access_key_id
                secret_access_key = args.secret_access_key
                region = args.region

            # Connect to AWS
            connection = connection_manager.connect_to_ec2(
                region, access_key_id, secret_access_key)

            snapshot_manager.run(connection)

            logger.info('Sleeping {} seconds until next check'.format(
                check_interval))
            time.sleep(check_interval)

Response: Run the daemon daemon


Instruction:
Can you implement a function in Python 3 that
iterates over the lines in a given markdown string and applies all the enabled line rules to each line.

Response:
def _apply_line_rules(self, markdown_string):
        """ Iterates over the lines in a given markdown string and applies all the enabled line rules to each line """
        all_violations = []
        lines = markdown_string.split("\n")
        line_rules = self.line_rules
        line_nr = 1
        ignoring = False
        for line in lines:
            if ignoring:
                if line.strip() == '<!-- markdownlint:enable -->':
                    ignoring = False
            else:
                if line.strip() == '<!-- markdownlint:disable -->':
                    ignoring = True
                    continue

                for rule in line_rules:
                    violation = rule.validate(line)
                    if violation:
                        violation.line_nr = line_nr
                        all_violations.append(violation)
            line_nr += 1
        return all_violations


Instruction:
How would you code a function in Python 3 to
lint a list of files.

Response:
def lint_files(self, files):
        """ Lints a list of files.
        :param files: list of files to lint
        :return: a list of violations found in the files
        """
        all_violations = []
        for filename in files:
            with open(filename, 'r') as f:
                content = f.read()
                violations = self.lint(content)
                all_violations.extend(violations)
                for e in violations:
                    print("{0}:{1}: {2} {3}".format(filename, e.line_nr, e.rule_id, e.message))
        return len(all_violations)


Instruction:
Write a Python 3 function for
reading the values from a byte stream.

Response:
def ReadFrom(self, byte_stream):
    """Read values from a byte stream.

    Args:
      byte_stream (bytes): byte stream.

    Returns:
      tuple[object, ...]: values copies from the byte stream.

    Raises:
      IOError: if byte stream cannot be read.
      OSError: if byte stream cannot be read.
    """
    try:
      return self._struct.unpack_from(byte_stream)
    except (TypeError, struct.error) as exception:
      raise IOError('Unable to read byte stream with error: {0!s}'.format(
          exception))


Instruction:
Can you generate the documentation for the following Python 3 function
def WriteTo(self, values):
    try:
      return self._struct.pack(*values)
    except (TypeError, struct.error) as exception:
      raise IOError('Unable to write stream with error: {0!s}'.format(
          exception))

Response: Writes values to a byte stream.


Instruction:
Write a Python 3 function that can
ensure that we have snapshots for a given volume.

Response:
def run(connection):
    """ Ensure that we have snapshots for a given volume

    :type connection: boto.ec2.connection.EC2Connection
    :param connection: EC2 connection object
    :returns: None
    """
    volumes = volume_manager.get_watched_volumes(connection)

    for volume in volumes:
        _ensure_snapshot(connection, volume)
        _remove_old_snapshots(connection, volume)


Instruction:
Can you implement a function in Python 3 that
creates a new snapshot for the given volume

Response:
def _create_snapshot(volume):
    """ Create a new snapshot

    :type volume: boto.ec2.volume.Volume
    :param volume: Volume to snapshot
    :returns: boto.ec2.snapshot.Snapshot -- The new snapshot
    """
    logger.info('Creating new snapshot for {}'.format(volume.id))
    snapshot = volume.create_snapshot(
        description="Automatic snapshot by Automated EBS Snapshots")
    logger.info('Created snapshot {} for volume {}'.format(
        snapshot.id, volume.id))

    return snapshot


Instruction:
Explain what the following Python 3 code does
def _ensure_snapshot(connection, volume):
    if 'AutomatedEBSSnapshots' not in volume.tags:
        logger.warning(
            'Missing tag AutomatedEBSSnapshots for volume {}'.format(
                volume.id))
        return

    interval = volume.tags['AutomatedEBSSnapshots']
    if volume.tags['AutomatedEBSSnapshots'] not in VALID_INTERVALS:
        logger.warning(
            '"{}" is not a valid snapshotting interval for volume {}'.format(
                interval, volume.id))
        return

    snapshots = connection.get_all_snapshots(filters={'volume-id': volume.id})

    # Create a snapshot if we don't have any
    if not snapshots:
        _create_snapshot(volume)
        return

    min_delta = 3600*24*365*10  # 10 years :)
    for snapshot in snapshots:
        timestamp = datetime.datetime.strptime(
            snapshot.start_time,
            '%Y-%m-%dT%H:%M:%S.000Z')
        delta_seconds = int(
            (datetime.datetime.utcnow() - timestamp).total_seconds())

        if delta_seconds < min_delta:
            min_delta = delta_seconds

    logger.info('The newest snapshot for {} is {} seconds old'.format(
        volume.id, min_delta))
    if interval == 'hourly' and min_delta > 3600:
        _create_snapshot(volume)
    elif interval == 'daily' and min_delta > 3600*24:
        _create_snapshot(volume)
    elif interval == 'weekly' and min_delta > 3600*24*7:
        _create_snapshot(volume)
    elif interval == 'monthly' and min_delta > 3600*24*30:
        _create_snapshot(volume)
    elif interval == 'yearly' and min_delta > 3600*24*365:
        _create_snapshot(volume)
    else:
        logger.info('No need for a new snapshot of {}'.format(volume.id))

Response: Ensures that a given volume has an appropriate snapshot.


Instruction:
Write a Python 3 script for
removing old snapshots from the list of old snapshots.

Response:
def _remove_old_snapshots(connection, volume):
    """ Remove old snapshots

    :type connection: boto.ec2.connection.EC2Connection
    :param connection: EC2 connection object
    :type volume: boto.ec2.volume.Volume
    :param volume: Volume to check
    :returns: None
    """
    if 'AutomatedEBSSnapshotsRetention' not in volume.tags:
        logger.warning(
            'Missing tag AutomatedEBSSnapshotsRetention for volume {}'.format(
                volume.id))
        return
    retention = int(volume.tags['AutomatedEBSSnapshotsRetention'])

    snapshots = connection.get_all_snapshots(filters={'volume-id': volume.id})

    # Sort the list based on the start time
    snapshots.sort(key=lambda x: x.start_time)

    # Remove snapshots we want to keep
    snapshots = snapshots[:-int(retention)]

    if not snapshots:
        logger.info('No old snapshots to remove')
        return

    for snapshot in snapshots:
        logger.info('Deleting snapshot {}'.format(snapshot.id))
        try:
            snapshot.delete()
        except EC2ResponseError as error:
            logger.warning('Could not remove snapshot: {}'.format(
                error.message))

    logger.info('Done deleting snapshots')


Instruction:
Implement a function in Python 3 to
list watched EBS volumes and return a new object.

Response:
def list(connection):
    """ List watched EBS volumes

    :type connection: boto.ec2.connection.EC2Connection
    :param connection: EC2 connection object
    :returns: None
    """
    volumes = get_watched_volumes(connection)

    if not volumes:
        logger.info('No watched volumes found')
        return

    logger.info(
        '+-----------------------'
        '+----------------------'
        '+--------------'
        '+------------+')
    logger.info(
        '| {volume:<21} '
        '| {volume_name:<20.20} '
        '| {interval:<12} '
        '| {retention:<10} |'.format(
            volume='Volume ID',
            volume_name='Volume name',
            interval='Interval',
            retention='Retention'))
    logger.info(
        '+-----------------------'
        '+----------------------'
        '+--------------'
        '+------------+')

    for volume in volumes:
        if 'AutomatedEBSSnapshots' not in volume.tags:
            interval = 'Interval tag not found'
        elif volume.tags['AutomatedEBSSnapshots'] not in VALID_INTERVALS:
            interval = 'Invalid interval'
        else:
            interval = volume.tags['AutomatedEBSSnapshots']

        if 'AutomatedEBSSnapshotsRetention' not in volume.tags:
            retention = 0
        else:
            retention = volume.tags['AutomatedEBSSnapshotsRetention']

        # Get the volume name
        try:
            volume_name = volume.tags['Name']
        except KeyError:
            volume_name = ''

        logger.info(
            '| {volume_id:<14} '
            '| {volume_name:<20.20} '
            '| {interval:<12} '
            '| {retention:<10} |'.format(
                volume_id=volume.id,
                volume_name=volume_name,
                interval=interval,
                retention=retention))

    logger.info(
        '+-----------------------'
        '+----------------------'
        '+--------------'
        '+------------+')


Instruction:
Can you tell what is the following Python 3 function doing
def unwatch(connection, volume_id):
    try:
        volume = connection.get_all_volumes(volume_ids=[volume_id])[0]
        volume.remove_tag('AutomatedEBSSnapshots')
    except EC2ResponseError:
        pass

    logger.info('Removed {} from the watchlist'.format(volume_id))

    return True

Response: Remove watching of a volume


Instruction:
Given the following Python 3 function, write the documentation
def watch(connection, volume_id, interval='daily', retention=0):
    try:
        volume = connection.get_all_volumes(volume_ids=[volume_id])[0]
    except EC2ResponseError:
        logger.warning('Volume {} not found'.format(volume_id))
        return False

    if interval not in VALID_INTERVALS:
        logger.warning(
            '{} is not a valid interval. Valid intervals are {}'.format(
                interval, ', '.join(VALID_INTERVALS)))

    # Remove the tag first
    volume.remove_tag('AutomatedEBSSnapshots')

    # Re-add the tag
    volume.add_tag('AutomatedEBSSnapshots', value=interval)

    # Remove the tag first
    volume.remove_tag('AutomatedEBSSnapshotsRetention')

    # Re-add the tag
    volume.add_tag('AutomatedEBSSnapshotsRetention', value=int(retention))

    logger.info('Updated the rotation interval to {} for {}'.format(
        interval, volume_id))

    return True

Response: Start watching a new volume and add it to the watchlist


Instruction:
How would you explain what the following Python 3 function does
def get_volume_id(connection, volume):
    # Regular expression to check whether input is a volume id
    volume_id_pattern = re.compile('vol-\w{8}')

    if volume_id_pattern.match(volume):
        # input is volume id
        try:
            # Check whether it exists
            connection.get_all_volumes(volume_ids=[volume])
            volume_id = volume
        except EC2ResponseError:
            logger.warning('Volume {} not found'.format(volume))
            return None
    else:
        # input is volume name
        name_filter = {'tag-key': 'Name', 'tag-value': volume}
        volumes = connection.get_all_volumes(filters=name_filter)
        if not volumes:
            logger.warning('Volume {} not found'.format(volume))
            return None
        if len(volumes) > 1:
            logger.warning('Volume {} not unique'.format(volume))
        volume_id = volumes[0].id

    return volume_id

Response:
Get Volume ID from the given volume. Input can be volume id
    or volume Name
   . Returns None if the given volume does not exist.


Instruction:
Explain what the following Python 3 code does
def watch_from_file(connection, file_name):
    with open(file_name, 'r') as filehandle:
        for line in filehandle.xreadlines():
            volume, interval, retention = line.rstrip().split(',')
            watch(
                connection,
                get_volume_id(connection, volume),
                interval, retention)

Response: Start watching a new block device from a file.


Instruction:
Can you create a Python 3 function that
starts watching a new volume :type connection: boto.ec2.connection.EC2Connection :param connection: EC2 connection object :type file_name: str :param file_name: path to config file :returns: None

Response:
def unwatch_from_file(connection, file_name):
    """ Start watching a new volume

    :type connection: boto.ec2.connection.EC2Connection
    :param connection: EC2 connection object
    :type file_name: str
    :param file_name: path to config file
    :returns: None
    """
    with open(file_name, 'r') as filehandle:
        for line in filehandle.xreadlines():
            volume, interval, retention = line.rstrip().split(',')
            unwatch(connection, get_volume_id(connection, volume))


Instruction:
How would you implement a function in Python 3 that
lists all snapshots for the specified volume

Response:
def list_snapshots(connection, volume):
    """ List all snapshots for the volume

    :type connection: boto.ec2.connection.EC2Connection
    :param connection: EC2 connection object
    :type volume: str
    :param volume: Volume ID or Volume Name
    :returns: None
    """

    logger.info(
        '+----------------'
        '+----------------------'
        '+---------------------------+')
    logger.info(
        '| {snapshot:<14} '
        '| {snapshot_name:<20.20} '
        '| {created:<25} |'.format(
            snapshot='Snapshot ID',
            snapshot_name='Snapshot name',
            created='Created'))
    logger.info(
        '+----------------'
        '+----------------------'
        '+---------------------------+')

    vid = get_volume_id(connection, volume)
    if vid:
        vol = connection.get_all_volumes(volume_ids=[vid])[0]
        for snap in vol.snapshots():
            logger.info(
                '| {snapshot:<14} '
                '| {snapshot_name:<20.20} '
                '| {created:<25} |'.format(
                    snapshot=snap.id,
                    snapshot_name=snap.tags.get('Name', ''),
                    created=snap.start_time))

    logger.info(
        '+----------------'
        '+----------------------'
        '+---------------------------+')


Instruction:
Create a Python 3 function for
getting stems for the words using a given parser

Response:
def stem(self, words, parser, **kwargs):
        """
        Get stems for the words using a given parser

        Example:
            from .parsing import ListParser

            parser = ListParser()
            stemmer = Morfologik()
            stemmer.stem(['ja tańczę a ona śpi], parser)

            [
                 ('ja': ['ja']),
                 ('tańczę': ['tańczyć']),
                 ('a': ['a']),
                 ('ona': ['on']),
                 ('śpi': ['spać'])
            ]
        """
        output = self._run_morfologik(words)
        return parser.parse(output, **kwargs)


Instruction:
Implement a Python 3 function for
running morfologik java jar and returns the result as unicode.

Response:
def _run_morfologik(self, words):
        """
        Runs morfologik java jar and assumes that input and output is
        UTF-8 encoded.
        """
        p = subprocess.Popen(
            ['java', '-jar', self.jar_path, 'plstem',
             '-ie', 'UTF-8',
             '-oe', 'UTF-8'],
            bufsize=-1,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT)
        out, _ = p.communicate(input=bytes("\n".join(words), "utf-8"))
        return decode(out, 'utf-8')


Instruction:
Can you generate the documentation for the following Python 3 function
def read_from_hdx(identifier, configuration=None):
        # type: (str, Optional[Configuration]) -> Optional['Showcase']
        """Reads the showcase given by identifier from HDX and returns Showcase object

        Args:
            identifier (str): Identifier of showcase
            configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration.

        Returns:
            Optional[Showcase]: Showcase object if successful read, None if not
        """

        showcase = Showcase(configuration=configuration)
        result = showcase._load_from_hdx('showcase', identifier)
        if result:
            return showcase
        return None

Response: Reads the showcase given by identifier from HDX and returns the object if successful.


Instruction:
Here you have a function in Python 3, explain what it does
def get_datasets(self):
        # type: () -> List[hdx.data.dataset.Dataset]
        """Get any datasets in the showcase

        Returns:
            List[Dataset]: List of datasets
        """
        assoc_result, datasets_dicts = self._read_from_hdx('showcase', self.data['id'], fieldname='showcase_id',
                                                           action=self.actions()['list_datasets'])
        datasets = list()
        if assoc_result:
            for dataset_dict in datasets_dicts:
                dataset = hdx.data.dataset.Dataset(dataset_dict, configuration=self.configuration)
                datasets.append(dataset)
        return datasets

Response: Get any datasets in the showcase


Instruction:
Write a Python 3 function for
getting showcase dataset dict from a showcase object or dict

Response:
def _get_showcase_dataset_dict(self, dataset):
        # type: (Union[hdx.data.dataset.Dataset,Dict,str]) -> Dict
        """Get showcase dataset dict

        Args:
            showcase (Union[Showcase,Dict,str]): Either a showcase id or Showcase metadata from a Showcase object or dictionary

        Returns:
            Dict: showcase dataset dict
        """
        if isinstance(dataset, hdx.data.dataset.Dataset) or isinstance(dataset, dict):
            if 'id' not in dataset:
                dataset = hdx.data.dataset.Dataset.read_from_hdx(dataset['name'])
            dataset = dataset['id']
        elif not isinstance(dataset, str):
            raise hdx.data.hdxobject.HDXError('Type %s cannot be added as a dataset!' % type(dataset).__name__)
        if is_valid_uuid(dataset) is False:
            raise hdx.data.hdxobject.HDXError('%s is not a valid dataset id!' % dataset)
        return {'showcase_id': self.data['id'], 'package_id': dataset}


Instruction:
Can you generate a brief explanation for the following Python 3 code
def add_dataset(self, dataset, datasets_to_check=None):
        # type: (Union[hdx.data.dataset.Dataset,Dict,str], List[hdx.data.dataset.Dataset]) -> bool
        """Add a dataset

        Args:
            dataset (Union[Dataset,Dict,str]): Either a dataset id or dataset metadata either from a Dataset object or a dictionary
            datasets_to_check (List[Dataset]): List of datasets against which to check existence of dataset. Defaults to datasets in showcase.

        Returns:
            bool: True if the dataset was added, False if already present
        """
        showcase_dataset = self._get_showcase_dataset_dict(dataset)
        if datasets_to_check is None:
            datasets_to_check = self.get_datasets()
        for dataset in datasets_to_check:
            if showcase_dataset['package_id'] == dataset['id']:
                return False
        self._write_to_hdx('associate', showcase_dataset, 'package_id')
        return True

Response: Adds a dataset to the showcase.


Instruction:
Write a Python 3 function that can
add multiple datasets to the showcase.

Response:
def add_datasets(self, datasets, datasets_to_check=None):
        # type: (List[Union[hdx.data.dataset.Dataset,Dict,str]], List[hdx.data.dataset.Dataset]) -> bool
        """Add multiple datasets

        Args:
            datasets (List[Union[Dataset,Dict,str]]): A list of either dataset ids or dataset metadata from Dataset objects or dictionaries
            datasets_to_check (List[Dataset]): List of datasets against which to check existence of dataset. Defaults to datasets in showcase.

        Returns:
            bool: True if all datasets added or False if any already present
        """
        if datasets_to_check is None:
            datasets_to_check = self.get_datasets()
        alldatasetsadded = True
        for dataset in datasets:
            if not self.add_dataset(dataset, datasets_to_check=datasets_to_check):
                alldatasetsadded = False
        return alldatasetsadded


Instruction:
Here you have a function in Python 3, explain what it does
def possible_parameter(nb, jsonable_parameter=True, end_cell_index=None):
    jh = _JupyterNotebookHelper(nb, jsonable_parameter, end_cell_index)

    if jsonable_parameter is True:
        PossibleParameter=collections.namedtuple('PossibleParameter',['name','value','cell_index'])
    else:
        PossibleParameter=collections.namedtuple('PossibleParameter',['name', 'cell_index'])

    res=[]
    for name, cell_index in jh.param_cell_index.items():
        if jsonable_parameter is True:
            res.append(PossibleParameter(name=name,value=jh.param_value[name],cell_index=cell_index))
        else:
            res.append(PossibleParameter(name=name,cell_index=cell_index))

    return sorted(res, key = lambda x: (x.name))

Response: Find the possible parameters from a jupyter notebook.


Instruction:
Can you write a function in Python 3 where it
runs an input jupyter notebook file and optionally (python3 only) parametrise it. One can pass arguments as keyword arguments or in a json format (file or string). For safety reasons, in order to avoid any code injection, only json serialisable keywords arguments are available. The keyword arguments are firstly encoded in json format using the standard json encoder. The json content is decoded into python objects using the standard json decoder and it is mapped to a variable assignment by unpacking it. The assignments are appended at the end of the cell where they are initially defined. Parameters ---------- input_path : str Path of the input jupyter notebook. output_path : str, optional Path of the output jupyter notebook. One can use the input_path location as relative path by starting with "///" . * can be used once in the beggining or end (excluding the ".ipynb" extension) as a wildcard of the input_path filename. "///_run_jnb/*-output" is the default value and states that the output is in the run_jnb folder with respect to the input_path directory and "-output" is appended to the input name. execution_path : str, optional The path of the folder where to execute the notebook. r'///input' or r'///output' can be used to denote the input / output folder. return_mode : ['parametrised_only','except',True, False], optional Flag to write the generated notebook to the output_path: "parametrised_only" writes the generated notebook without executing it, "except" writes in case of an exception, True writes always, False writes never. overwrite : bool, optional Flag to overwrite or not the output_path. If the parameter is False the used output_path will be incremented until a valid one is found. timeout : int, optional ExecutePreprocessor.timeout kernel_name : str, optional ExecutePreprocessor.kernel_name ep_kwargs : dict, optional Other kwargs accepted by nbconvert.preprocessors.ExecutePreprocessor jsonable_parameter: bool, optional Parametrise only jsonable parameters end_cell_index : int, optional End cell index used to slice the notebook in finding the possible parameters. arg : str Path of a json file (it should end in ".json") or json formatted string used to parametrise the jupyter notebook. It should containt json objects. It is decoded into python objects following https://docs.python.org/3.6/library/json.html#json-to-py-table . kwargs: json serialsable keyword arguments used to parametrise the jupyter notebook. Returns ------- collections.namedtuple The fields are ('output_nb_path', 'error_prompt_number','error_type','error_value','error_traceback'). If the generated file is written the output path is returned otherwise None. If an error is catched the details are return otherwise None.

Response:
def run_jnb(input_path, output_path=r"///_run_jnb/*-output",
            execution_path=r'///input',
            return_mode='except',
            overwrite=False,
            timeout=ExecutePreprocessor.timeout.default_value,
            kernel_name=ExecutePreprocessor.kernel_name.default_value,
            ep_kwargs=None, jsonable_parameter=True, end_cell_index=None, arg=None, **kwargs):
    """
    Run an input jupyter notebook file and optionally (python3 only)
    parametrise it.

    One can pass arguments as keyword arguments or in a json format (file or string).
    For safety reasons, in order to avoid any code injection,
    only json serialisable keywords arguments are available. The keyword
    arguments are firstly encoded in json format using the standard json encoder.
    The json content is decoded into python objects using the standard json decoder
    and it is mapped to a variable assignment by unpacking it.
    The assignments are appended at the end of the cell where they are initially defined.

    Parameters
    ----------
    input_path : str
        Path of the input jupyter notebook.
    output_path : str, optional
        Path of the output jupyter notebook.
        One can use the input_path location as relative path by starting with "///" .
        * can be used once in the beggining or end (excluding the ".ipynb" extension) as a wildcard of the input_path filename.
        "///_run_jnb/*-output" is the default value and states that the output
        is in the run_jnb folder with respect to the input_path directory and "-output" is appended to the input name.
    execution_path : str, optional
        The path of the folder where to execute the notebook.
        r'///input' or r'///output' can be used to denote the input / output folder.

    return_mode : ['parametrised_only','except',True, False], optional
        Flag to write the generated notebook to the output_path: "parametrised_only" writes the generated notebook without executing it, "except" writes in case of an exception, True writes always, False writes never.
    overwrite : bool, optional
        Flag to overwrite or not the output_path. If the parameter is False
        the used output_path will be incremented until a valid one is found.
    timeout : int, optional
        ExecutePreprocessor.timeout
    kernel_name : str, optional
        ExecutePreprocessor.kernel_name
    ep_kwargs : dict, optional
        Other kwargs accepted by nbconvert.preprocessors.ExecutePreprocessor
    jsonable_parameter: bool, optional
        Parametrise only jsonable parameters
    end_cell_index : int, optional
        End cell index used to slice the notebook in finding the possible parameters.
    arg : str
        Path of a json file (it should end in ".json") or json formatted string used to parametrise the jupyter notebook.
        It should containt json objects. It is decoded into python objects following https://docs.python.org/3.6/library/json.html#json-to-py-table .
    kwargs:
        json serialsable keyword arguments used to parametrise the jupyter notebook.

    Returns
    -------
    collections.namedtuple
        The fields are ('output_nb_path', 'error_prompt_number','error_type','error_value','error_traceback').
        If the generated file is written the output path is returned otherwise None.
        If an error is catched the details are return otherwise None.
        """

    if os.path.splitext(input_path)[1] != '.ipynb':
        raise ValueError("The extension of input_path = '{}' is not '.ipynb'".format(input_path))
    if os.path.basename(input_path) == '*':
        raise ValueError("The filename ={} can not start with *".format(input_path))

    input_path_dir, input_path_base = os.path.split(input_path)
    if output_path.startswith(r'///'):
        input_rel = True
        output_path = output_path[3:]
    else:
        input_rel = False
    output_path_dir, output_path_base = os.path.split(output_path)

    if input_rel:
        output_path_dir = os.path.join(input_path_dir, output_path_dir)
        output_path_dir = os.path.normpath(output_path_dir)

    if os.path.exists(output_path_dir) is False:
        os.makedirs(output_path_dir)

    if output_path_base.endswith('.ipynb'):
        pass
    elif output_path_base.startswith("*"):
        output_path_base = input_path_base[:-6]+output_path_base[1:]+'.ipynb'
    elif output_path_base.endswith("*"):
        output_path_base = output_path_base[:-1]+input_path_base[:-6]+'.ipynb'
    else:
        raise ValueError("Invalid output_path")

    output_path = os.path.abspath(os.path.join(output_path_dir,
                                               output_path_base))

    if output_path is not None and os.path.splitext(output_path)[1] != '.ipynb':
        raise ValueError("The extension of output_path = '{}' is not '.ipynb'".format(output_path))

    if execution_path.startswith(r'///input'):
        execution_path = os.path.join(input_path_dir, execution_path[8:])
    elif execution_path.startswith(r'///output'):
        execution_path = os.path.join(input_path_dir, execution_path[8:])
    execution_path = os.path.normpath(execution_path)

    if os.path.exists(execution_path) is False:
        os.makedirs(execution_path)

    if ep_kwargs is None:
        ep_kwargs = {}

    if return_mode not in ['parametrised_only', 'except', True, False]:
        raise TypeError("return mode is not valid!")

    kwarg_to_json = json.dumps(kwargs)
    kwarg_as_kwarg = decode_json(kwarg_to_json)
    arg_as_kwarg = decode_json(arg)

    multiple_kwarg = set(arg_as_kwarg.keys()) & set(kwarg_as_kwarg.keys())
    if multiple_kwarg != set():
        raise ValueError('Multiple values for keyword argument {}'.format(multiple_kwarg))
    jupyter_kwargs = {**arg_as_kwarg, **kwarg_as_kwarg}
    nb = _read_nb(input_path)

    # clean notebook
    for i, cell in enumerate(nb['cells']):
        if cell['cell_type'] == 'code':
            nb['cells'][i]['outputs'] = []
            nb['cells'][i]['execution_count'] = None

    if jupyter_kwargs != {}:
        params_of_interest = {}
        jnh = _JupyterNotebookHelper(nb, jsonable_parameter ,end_cell_index)
        for el in jupyter_kwargs.keys():
            if el not in jnh.param_cell_index.keys():
                raise ValueError(repr(el)+' is not a possible parameter {}.'.format(list(jnh.param_cell_index.keys())))
            else:
                params_of_interest[el] = jnh.param_cell_index[el]
        params_of_interest = sort_dict(params_of_interest, by='value')
        cell_index_param = group_dict_by_value(params_of_interest)
        for key, value in cell_index_param.items():
            cell_param = {k: jupyter_kwargs[k] for k in value}
            cell_code = kwargs_to_variable_assignment(cell_param)
            marked_code = _mark_auto_generated_code(cell_code)
            nb['cells'][key]['source'] += marked_code

    if return_mode != 'parametrised_only':
        ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel_name,
                                 **ep_kwargs)

    catch_except = False

    error = (None, None, None, None)
    try:
        if return_mode != 'parametrised_only':
            ep.preprocess(nb, {'metadata': {'path': execution_path}})
    except CellExecutionError:
        catch_except = True

        for cell in nb['cells'][::-1]:
            if cell['cell_type'] == 'code' and cell.get('outputs') != []:
                for output in cell['outputs']:
                    if output.get('output_type') == 'error':
                        error = (cell['execution_count'],
                                 output.get('ename'), output.get('evalue'),
                                 output.get('traceback'))
                        break
                if error[0] is not None:
                    break
                else:
                    raise ValueError('Cell expected to have an error.')
    except:
        raise

    if return_mode == 'except':
        if catch_except is True:
            nb_return = True
        else:
            nb_return = None
    elif return_mode is True or return_mode == 'parametrised_only':
        nb_return = True
    elif return_mode is False:
        nb_return = None

    if nb_return is not None:
        if overwrite is False:
            while os.path.exists(output_path):
                dirname, basename = os.path.split(output_path)
                root, ext = os.path.splitext(basename)
                new_root = increment_name(root)
                output_path = os.path.join(dirname, new_root+ext)
        nb_return = output_path  # update the output_path
        _write_nb(nb, output_path)
    Output=collections.namedtuple('Output',['output_nb_path', 'error_prompt_number','error_type','error_value','error_traceback'])
    res = Output(output_nb_path=nb_return,error_prompt_number=error[0],
                error_type=error[1],error_value=error[2],error_traceback=error[3])
    return res


Instruction:
Can you generate a brief explanation for the following Python 3 code
def join(self, joiner, formatter=lambda s, t: t.format(s),
             template="{}"):
        """Join values and convert to string

        Example:

            >>> from ww import l
            >>> lst = l('012')
            >>> lst.join(',')
            u'0,1,2'
            >>> lst.join(',', template="{}#")
            u'0#,1#,2#'
            >>> string = lst.join(',',\
                                  formatter = lambda x, y: str(int(x) ** 2))
            >>> string
            u'0,1,4'
        """

        return ww.s(joiner).join(self, formatter, template)

Response: Join values and convert to string


Instruction:
Here you have a function in Python 3, explain what it does
def append(self, *values):

        for value in values:
            list.append(self, value)
        return self

Response: Append values at the end of the list.


Instruction:
How would you explain what the following Python 3 function does
def extend(self, *iterables):

        for value in iterables:
            list.extend(self, value)
        return self

Response: Add all values of all iterables at the end of the list


Instruction:
Write a Python 3 script for
normalizing the value for writing into a cell.

Response:
def normalize_cell_value(value):
    """Process value for writing into a cell.

    Args:
        value: any type of variable

    Returns:
        json serialized value if value is list or dict, else value
    """
    if isinstance(value, dict) or isinstance(value, list):
        return json.dumps(value)
    return value


Instruction:
Make a summary of the following Python 3 code
def get_addresses_from_input_file(input_file_name):
    mode = 'r'
    if sys.version_info[0] < 3:
        mode = 'rb'
    with io.open(input_file_name, mode) as input_file:
        reader = csv.reader(input_file, delimiter=',', quotechar='"')

        addresses = list(map(tuple, reader))

        if len(addresses) == 0:
            raise Exception('No addresses found in input file')

        header_columns = list(column.lower() for column in addresses.pop(0))

        try:
            address_index = header_columns.index('address')
            zipcode_index = header_columns.index('zipcode')
        except ValueError:
            raise Exception("""The first row of the input CSV must be a header that contains \
a column labeled 'address' and a column labeled 'zipcode'.""")

        return list((row[address_index], row[zipcode_index]) for row in addresses)

Response: Read addresses from input file into list of tuples. This only supports address and zipcode headers.


Instruction:
Here you have a function in Python 3, explain what it does
def get_identifiers_from_input_file(input_file_name):
    valid_identifiers = ['address', 'zipcode', 'unit', 'city', 'state', 'slug', 'block_id', 'msa',
                         'num_bins', 'property_type', 'client_value', 'client_value_sqft', 'meta']
    mode = 'r'
    if sys.version_info[0] < 3:
        mode = 'rb'
    with io.open(input_file_name, mode) as input_file:
        result = [{identifier: val for identifier, val in list(row.items())
                   if identifier in valid_identifiers}
                  for row in csv.DictReader(input_file, skipinitialspace=True)]
        return result

Response:
Read identifiers from input file into list of dicts with the header row values
ATTRIBS as keys and the rest of the rows as values.


Instruction:
Here you have a function in Python 3, explain what it does
def get_real_time_locate(ipAddress, auth, url):
    real_time_locate_url = "/imcrs/res/access/realtimeLocate?type=2&value=" + str(ipAddress) + "&total=false"
    f_url = url + real_time_locate_url
    r = requests.get(f_url, auth=auth, headers=HEADERS)  # creates the URL using the payload variable as the contents
    try:
        if r.status_code == 200:
            response =  json.loads(r.text)
            if 'realtimeLocation' in response:
                real_time_locate = json.loads(r.text)['realtimeLocation']
                if type(real_time_locate) is dict:
                    real_time_locate = [real_time_locate]
                    return real_time_locate
                else:
                    return json.loads(r.text)['realtimeLocation']
            else:
                return json.loads(r.text)

    except requests.exceptions.RequestException as e:
            return "Error:\n" + str(e) + " get_real_time_locate: An Error has occured"

Response: This function returns the real time locate of the specified host.


Instruction:
Write a Python 3 script to
function takes devId of specific device and issues a RESTFUL call to get the IP MAC ARP list of the target device.

Response:
def get_ip_mac_arp_list(devId, auth,url):
    """
    function takes devid of specific device and issues a RESTFUL call to get the IP/MAC/ARP list from the target device.

    :param devId: int or str value of the target device.

    :param auth: requests auth object #usually auth.creds from auth pyhpeimc.auth.class

    :param url: base url of IMC RS interface #usually auth.url from pyhpeimc.auth.authclass

    :return: list of dictionaries containing the IP/MAC/ARP list of the target device.

    :rtype: list

    >>> from pyhpeimc.auth import *

    >>> from pyhpeimc.plat.termaccess import *

    >>> auth = IMCAuth("http://", "10.101.0.203", "8080", "admin", "admin")

    >>> ip_mac_list = get_ip_mac_arp_list('10', auth.creds, auth.url)

    >>> assert type(ip_mac_list) is list

    >>> assert 'deviceId' in ip_mac_list[0]

    """
    if auth is None or url is None:  # checks to see if the imc credentials are already available
        set_imc_creds()
    ip_mac_arp_list_url = "/imcrs/res/access/ipMacArp/" + str(devId)
    f_url = url + ip_mac_arp_list_url
    r = requests.get(f_url, auth=auth, headers=HEADERS)  # creates the URL using the payload variable as the contents
    try:
        if r.status_code == 200:
            macarplist = (json.loads(r.text))
            if len(macarplist) > 1:
                return macarplist['ipMacArp']
            else:
                return ['this function is unsupported']

    except requests.exceptions.RequestException as e:
            return "Error:\n" + str(e) + " get_ip_mac_arp_list: An Error has occured"


Instruction:
How would you explain what the following Python 3 function does
def get_ip_scope_detail(scopeId, auth, url ):
    get_ip_scope_url = "/imcrs/res/access/assignedIpScope/"+str(scopeId)

    f_url = url + get_ip_scope_url
    r = requests.get(f_url, auth=auth, headers=HEADERS)  # creates the URL using the payload variable as the contents
    try:
        if r.status_code == 200:
            ipscopelist = (json.loads(r.text))
            return ipscopelist
    except requests.exceptions.RequestException as e:
            return "Error:\n" + str(e) + " get_ip_scope: An Error has occured"

Response: function takes input of scopeId and auth object and url and returns the list of IP address scopes currently configured on the HPE IMC server.


Instruction:
Write a Python 3 script for
functioning to delete an entire IP segment from the IMC IP Address management under terminal access

Response:
def delete_ip_scope(network_address, auth, url):
    '''Function to delete an entire IP segment from the IMC IP Address management under terminal access
    :param network_address
    :param auth
    :param url

    >>> from pyhpeimc.auth import *

    >>> from pyhpeimc.plat.termaccess import *

    >>> auth = IMCAuth("http://", "10.101.0.203", "8080", "admin", "admin")

    >>> new_scope = add_ip_scope('10.50.0.1', '10.50.0.254', 'cyoung', 'test group', auth.creds, auth.url)

    >>> delete_scope = delete_ip_scope('10.50.0.0/24', auth.creds, auth.url)

    '''
    scope_id = get_scope_id(network_address, auth,url)
    delete_ip_address_url = '''/imcrs/res/access/assignedIpScope/'''+str(scope_id)
    f_url = url + delete_ip_address_url
    r = requests.delete(f_url, auth=auth, headers=HEADERS)
    try:
        return r
        if r.status_code == 204:
            #print("IP Segment Successfully Deleted")
            return r.status_code
    except requests.exceptions.RequestException as e:
        return "Error:\n" + str(e) + " delete_ip_scope: An Error has occured"


Instruction:
Can you implement a function in Python 3 that
functions to add new host IP address allocation to existing scope ID

Response:
def add_scope_ip(ipaddress, name, description, scopeid, auth, url):
    """
    Function to add new host IP address allocation to existing scope ID

    :param ipaddress:

    :param name: name of the owner of this host

    :param description: Description of the host

    :param auth: requests auth object #usually auth.creds from auth pyhpeimc.auth.class

    :param url: base url of IMC RS interface #usually auth.url from pyhpeimc.auth.authclass

    :return:

    :rtype:

    >>> from pyhpeimc.auth import *

    >>> from pyhpeimc.plat.termaccess import *

    >>> auth = IMCAuth("http://", "10.101.0.203", "8080", "admin", "admin")

    >>> add_scope_ip('10.50.0.5', 'cyoung', 'New Test Host','175', auth.creds, auth.url)

    """
    new_ip = { "ip": ipaddress,
      "name": name,
      "description": description}
    add_scope_ip_url = '/imcrs/res/access/assignedIpScope/ip?ipScopeId='+str(scopeid)
    f_url = url + add_scope_ip_url
    payload = json.dumps(new_ip)
    r = requests.post(f_url, auth=auth, headers=HEADERS,
                      data=payload)  # creates the URL using the payload variable as the contents
    try:
        if r.status_code == 200:
            #print("IP Scope Successfully Created")
            return r.status_code
        elif r.status_code == 409:
            #print("IP Scope Already Exists")
            return r.status_code
    except requests.exceptions.RequestException as e:
        return "Error:\n" + str(e) + " add_ip_scope: An Error has occured"


Instruction:
Can you generate the documentation for the following Python 3 function
def remove_scope_ip(hostid, auth, url):
    add_scope_ip_url = '/imcrs/res/access/assignedIpScope/ip/'+str(hostid)
    f_url = url + add_scope_ip_url

    r = requests.delete(f_url, auth=auth, headers=HEADERS,
                      )
    try:
        if r.status_code == 204:
            #print("Host Successfully Deleted")
            return r.status_code
        elif r.status_code == 409:
            #print("IP Scope Already Exists")
            return r.status_code
    except requests.exceptions.RequestException as e:
        return "Error:\n" + str(e) + " add_ip_scope: An Error has occured"

Response: Function to add remove IP address allocation to the base url of the host