Instruction:
Can you tell what is the following Python 3 function doing
def _load_ini_based_io(path, recursive=False, ini=None,
                       subini={}, include_core=True,
                       only_coefficients=False):
    """ DEPRECATED: For convert a previous version to the new json format

    Loads a IOSystem or Extension from a ini files

    This function can be used to load a IOSystem or Extension specified in a
    ini file. DataFrames (tables) are loaded from text or binary pickle files.
    For the latter, the extension .pkl or .pickle is assumed, in all other case
    the tables are assumed to be in .txt format.

    Parameters
    ----------

    path : string
        path or ini file name for the data to load

    recursive : boolean, optional
        If True, load also the data in the subfolders and add them as
        extensions to the IOSystem (in that case path must point to the root).
        Only first order subfolders are considered (no subfolders in
        subfolders) and if a folder does not contain a ini file it's skipped.
        Use the subini parameter in case of multiple ini files in a subfolder.
        Attribute name of the extension in the IOSystem are based on the
        subfolder name.  Default is False

    ini : string, optional
        If there are several ini files in the root folder, take this one for
        loading the data If None (default) take the ini found in the folder,
        error if several are found

    subini : dict, optional
        If there are multiple ini in the subfolder, use the ini given in the
        dict.  Format: 'subfoldername':'ininame' If a key for a subfolder is
        not found or None (default), the ini found in the folder will be taken,
        error if several are found

    include_core : boolean, optional
        If False the load method does not include A, L and Z matrix. This
        significantly reduces the required memory if the purpose is only
        to analyse the results calculated beforehand.

    Returns
    -------

        IOSystem or Extension class depending on systemtype in the ini file
        None in case of errors

    """

    # check path and given parameter
    ini_file_name = None

    path = os.path.abspath(os.path.normpath(path))

    if os.path.splitext(path)[1] == '.ini':
        (path, ini_file_name) = os.path.split(path)

    if ini:
        ini_file_name = ini

    if not os.path.exists(path):
        raise ReadError('Given path does not exist')
        return None

    if not ini_file_name:
        _inifound = False
        for file in os.listdir(path):
            if os.path.splitext(file)[1] == '.ini':
                if _inifound:
                    raise ReadError(
                            'Found multiple ini files in folder - specify one')
                    return None
                ini_file_name = file
                _inifound = True

    # read the ini
    io_ini = configparser.RawConfigParser()
    io_ini.optionxform = lambda option: option

    io_ini.read(os.path.join(path, ini_file_name))

    systemtype = io_ini.get('systemtype', 'systemtype', fallback=None)
    name = io_ini.get('meta', 'name',
                      fallback=os.path.splitext(ini_file_name)[0])

    if systemtype == 'IOSystem':
        ret_system = IOSystem(name=name)
    elif systemtype == 'Extension':
        ret_system = Extension(name=name)
    else:
        raise ReadError('System not defined in ini')
        return None

    for key in io_ini['meta']:
        setattr(ret_system, key, io_ini.get('meta', key, fallback=None))

    for key in io_ini['files']:
        if '_nr_index_col' in key:
            continue
        if '_nr_header' in key:
            continue

        if not include_core:
            not_to_load = ['A', 'L', 'Z']
            if key in not_to_load:
                continue

        if only_coefficients:
            _io = IOSystem()
            if key not in _io.__coefficients__ + ['unit']:
                continue

        file_name = io_ini.get('files', key)
        nr_index_col = io_ini.get(
            'files', key + '_nr_index_col', fallback=None)
        nr_header = io_ini.get('files', key + '_nr_header', fallback=None)

        if (nr_index_col is None) or (nr_header is None):
            raise ReadError(
                    'Index or column specification missing for {}'.
                    format(str(file_name)))
            return None

        _index_col = list(range(int(nr_index_col)))
        _header = list(range(int(nr_header)))

        if _index_col == [0]:
            _index_col = 0
        if _header == [0]:
            _header = 0
        file = os.path.join(path, file_name)
        logging.info('Load data from {}'.format(file))

        if (os.path.splitext(file)[1] == '.pkl' or
                os.path.splitext(file)[1] == '.pickle'):
            setattr(ret_system, key,
                    pd.read_pickle(file))
        else:
            setattr(ret_system, key,
                    pd.read_table(file,
                                  index_col=_index_col,
                                  header=_header))

    if recursive:
        # look for subfolder in the given path
        subfolder_list = os.walk(path).__next__()[1]

        # loop all subfolder and append extension based on
        # ini file in subfolder
        for subfolder in subfolder_list:
            subini_file_name = subini.get(subfolder)
            subpath = os.path.abspath(os.path.join(path, subfolder))

            if not subini_file_name:
                _inifound = False
                for file in os.listdir(subpath):
                    if os.path.splitext(file)[1] == '.ini':
                        if _inifound:
                            raise ReadError(
                                'Found multiple ini files in subfolder '
                                '{} - specify one'.format(subpath))
                            return None
                        subini_file_name = file
                        _inifound = True
            if not _inifound:
                continue

            # read the ini
            subio_ini = configparser.RawConfigParser()
            subio_ini.optionxform = lambda option: option

            subio_ini.read(os.path.join(subpath, subini_file_name))

            systemtype = subio_ini.get('systemtype', 'systemtype',
                                       fallback=None)
            name = subio_ini.get('meta', 'name',
                                 fallback=os.path.splitext(
                                     subini_file_name)[0])

            if systemtype == 'IOSystem':
                raise ReadError('IOSystem found in subfolder {} - '
                                'only extensions expected'.format(subpath))
                return None
            elif systemtype == 'Extension':
                sub_system = Extension(name=name)
            else:
                raise ReadError('System not defined in ini')
                return None

            for key in subio_ini['meta']:
                setattr(sub_system, key, subio_ini.get('meta', key,
                                                       fallback=None))

            for key in subio_ini['files']:
                if '_nr_index_col' in key:
                    continue
                if '_nr_header' in key:
                    continue

                if only_coefficients:
                    _ext = Extension('temp')
                    if key not in _ext.__coefficients__ + ['unit']:
                        continue

                file_name = subio_ini.get('files', key)
                nr_index_col = subio_ini.get('files', key + '_nr_index_col',
                                             fallback=None)
                nr_header = subio_ini.get('files', key + '_nr_header',
                                          fallback=None)

                if (nr_index_col is None) or (nr_header is None):
                    raise ReadError('Index or column specification missing '
                                    'for {}'.format(str(file_name)))
                    return None

                _index_col = list(range(int(nr_index_col)))
                _header = list(range(int(nr_header)))

                if _index_col == [0]:
                    _index_col = 0
                if _header == [0]:
                    _header = 0
                file = os.path.join(subpath, file_name)
                logging.info('Load data from {}'.format(file))
                if (os.path.splitext(file)[1] == '.pkl' or
                        os.path.splitext(file)[1] == '.pickle'):
                    setattr(sub_system, key,
                            pd.read_pickle(file))
                else:
                    setattr(sub_system, key,
                            pd.read_table(file,
                                          index_col=_index_col,
                                          header=_header))

                # get valid python name from folder
                def clean(varStr):
                    return re.sub('\W|^(?=\d)', '_', str(varStr))

                setattr(ret_system, clean(subfolder), sub_system)

    return ret_system

Response: This function loads a new json file from a ini file.


Instruction:
Implement a Python 3 function for
parsing an EXIOBASE version 1 or 2 like extension file into a single entry in the index.

Response:
def parse_exio12_ext(ext_file, index_col, name, drop_compartment=True,
                     version=None, year=None, iosystem=None, sep=','):
    """ Parse an EXIOBASE version 1 or 2 like extension file into pymrio.Extension

    EXIOBASE like extensions files are assumed to have two
    rows which are used as columns multiindex (region and sector)
    and up to three columns for the row index (see Parameters).

    For EXIOBASE 3 - extension can be loaded directly with pymrio.load

    Notes
    -----
    So far this only parses factor of production extensions F (not
    final demand extensions FY nor coeffiecents S).

    Parameters
    ----------

    ext_file : string or pathlib.Path
        File to parse

    index_col : int
        The number of columns (1 to 3) at the beginning of the file
        to use as the index. The order of the index_col must be
        - 1 index column: ['stressor']
        - 2 index columns: ['stressor', 'unit']
        - 3 index columns: ['stressor', 'compartment', 'unit']
        - > 3: everything up to three index columns will be removed

    name : string
        Name of the extension

    drop_compartment : boolean, optional
        If True (default) removes the compartment from the index.

    version : string, optional
        see pymrio.Extension

    iosystem : string, optional
        see pymrio.Extension

    year : string or int
        see pymrio.Extension

    sep : string, optional
        Delimiter to use; default ','

    Returns
    -------
    pymrio.Extension
        with F (and unit if available)

    """

    ext_file = os.path.abspath(str(ext_file))

    F = pd.read_table(
        ext_file,
        header=[0, 1],
        index_col=list(range(index_col)),
        sep=sep)

    F.columns.names = ['region', 'sector']

    if index_col == 1:
        F.index.names = ['stressor']

    elif index_col == 2:
        F.index.names = ['stressor', 'unit']

    elif index_col == 3:
        F.index.names = ['stressor', 'compartment', 'unit']

    else:
        F.reset_index(level=list(range(3, index_col)),
                      drop=True,
                      inplace=True)
        F.index.names = ['stressor', 'compartment', 'unit']

    unit = None
    if index_col > 1:
        unit = pd.DataFrame(F.iloc[:, 0].
                            reset_index(level='unit').unit)
        F.reset_index(level='unit', drop=True, inplace=True)

    if drop_compartment:
        F.reset_index(level='compartment',
                      drop=True, inplace=True)
        unit.reset_index(level='compartment',
                         drop=True, inplace=True)

    return Extension(name=name,
                     F=F,
                     unit=unit,
                     iosystem=iosystem,
                     version=version,
                     year=year,
                     )


Instruction:
How would you implement a function in Python 3 that
returns the EXIOBASE version for the given filename None if not found

Response:
def get_exiobase12_version(filename):
    """ Returns the EXIOBASE version for the given filename,
        None if not found
    """
    try:
        ver_match = re.search(r'(\d+\w*(\.|\-|\_))*\d+\w*', filename)
        version = ver_match.string[ver_match.start():ver_match.end()]
        if re.search('\_\d\d\d\d', version[-5:]):
            version = version[:-5]
    except AttributeError:
        version = None

    return version


Instruction:
Here you have a function in Python 3, explain what it does
def get_exiobase_files(path, coefficients=True):
    path = os.path.normpath(str(path))
    if coefficients:
        exio_core_regex = dict(
            # don’t match file if starting with _
            A=re.compile('(?<!\_)mrIot.*txt'),
            Y=re.compile('(?<!\_)mrFinalDemand.*txt'),
            S_factor_inputs=re.compile('(?<!\_)mrFactorInputs.*txt'),
            S_emissions=re.compile('(?<!\_)mrEmissions.*txt'),
            S_materials=re.compile('(?<!\_)mrMaterials.*txt'),
            S_resources=re.compile('(?<!\_)mrResources.*txt'),
            FY_resources=re.compile('(?<!\_)mrFDResources.*txt'),
            FY_emissions=re.compile('(?<!\_)mrFDEmissions.*txt'),
            FY_materials=re.compile('(?<!\_)mrFDMaterials.*txt'),
            )
    else:
        exio_core_regex = dict(
            # don’t match file if starting with _
            Z=re.compile('(?<!\_)mrIot.*txt'),
            Y=re.compile('(?<!\_)mrFinalDemand.*txt'),
            F_fac=re.compile('(?<!\_)mrFactorInputs.*txt'),
            F_emissions=re.compile('(?<!\_)mrEmissions.*txt'),
            F_materials=re.compile('(?<!\_)mrMaterials.*txt'),
            F_resources=re.compile('(?<!\_)mrResources.*txt'),
            FY_emissions=re.compile('(?<!\_)mrFDEmissions.*txt'),
            FY_materials=re.compile('(?<!\_)mrFDMaterials.*txt'),
            )

    repo_content = get_repo_content(path)

    exio_files = dict()
    for kk, vv in exio_core_regex.items():
        found_file = [vv.search(ff).string for ff in repo_content.filelist
                      if vv.search(ff)]
        if len(found_file) > 1:
            logging.warning(
                "Multiple files found for {}: {}"
                " - USING THE FIRST ONE".format(kk, found_file))
            found_file = found_file[0:1]
        elif len(found_file) == 0:
            continue
        else:
            if repo_content.iszip:
                format_para = sniff_csv_format(found_file[0],
                                               zip_file=path)
            else:
                format_para = sniff_csv_format(os.path.join(path,
                                                            found_file[0]))
            exio_files[kk] = dict(
                root_repo=path,
                file_path=found_file[0],
                version=get_exiobase12_version(
                    os.path.basename(found_file[0])),
                index_rows=format_para['nr_header_row'],
                index_col=format_para['nr_index_col'],
                unit_col=format_para['nr_index_col'] - 1,
                sep=format_para['sep'])

    return exio_files

Response: Returns a dict of all EXIOBASE files in path.


Instruction:
Can you generate the documentation for the following Python 3 function
def generic_exiobase12_parser(exio_files, system=None):

    version = ' & '.join({dd.get('version', '')
                          for dd in exio_files.values()
                          if dd.get('version', '')})

    meta_rec = MRIOMetaData(system=system,
                            name="EXIOBASE",
                            version=version)

    if len(version) == 0:
        meta_rec.note("No version information found, assuming exiobase 1")
        meta_rec.change_meta('version', 1)
        version = '1'

    core_components = ['A', 'Y', 'Z']

    core_data = dict()
    ext_data = dict()
    for tt, tpara in exio_files.items():
        full_file_path = os.path.join(tpara['root_repo'], tpara['file_path'])
        logging.debug("Parse {}".format(full_file_path))
        if tpara['root_repo'][-3:] == 'zip':
            with zipfile.ZipFile(tpara['root_repo'], 'r') as zz:
                raw_data = pd.read_table(
                    zz.open(tpara['file_path']),
                    index_col=list(range(tpara['index_col'])),
                    header=list(range(tpara['index_rows'])))
        else:
            raw_data = pd.read_table(
                full_file_path,
                index_col=list(range(tpara['index_col'])),
                header=list(range(tpara['index_rows'])))

        meta_rec._add_fileio('EXIOBASE data {} parsed from {}'.format(
            tt, full_file_path))
        if tt in core_components:
            core_data[tt] = raw_data
        else:
            ext_data[tt] = raw_data

    for table in core_data:
        core_data[table].index.names = ['region', 'sector', 'unit']
        if table == 'A' or table == 'Z':
            core_data[table].columns.names = ['region', 'sector']
            _unit = pd.DataFrame(
                    core_data[table].iloc[:, 0]).reset_index(
                        level='unit').unit
            _unit = pd.DataFrame(_unit)
            _unit.columns = ['unit']
        if table == 'Y':
            core_data[table].columns.names = ['region', 'category']
        core_data[table].reset_index(level='unit', drop=True, inplace=True)

    core_data['unit'] = _unit

    mon_unit = core_data['unit'].iloc[0, 0]
    if '/' in mon_unit:
        mon_unit = mon_unit.split('/')[0]
        core_data['unit'].unit = mon_unit

    extensions = dict()
    for tt, tpara in exio_files.items():
        if tt in core_components:
            continue
        ext_name = '_'.join(tt.split('_')[1:])
        table_type = tt.split('_')[0]

        if tpara['index_col'] == 3:
            ext_data[tt].index.names = [
                'stressor', 'compartment', 'unit']
        elif tpara['index_col'] == 2:
            ext_data[tt].index.names = [
                'stressor', 'unit']
        else:
            raise ParserError('Unknown EXIOBASE file structure')

        if table_type == 'FY':
            ext_data[tt].columns.names = [
                'region', 'category']
        else:
            ext_data[tt].columns.names = [
                'region', 'sector']
        try:
            _unit = pd.DataFrame(
                ext_data[tt].iloc[:, 0]
            ).reset_index(level='unit').unit
        except IndexError:
            _unit = pd.DataFrame(
                ext_data[tt].iloc[:, 0])
            _unit.columns = ['unit']
            _unit['unit'] = 'undef'
            _unit.reset_index(level='unit', drop=True, inplace=True)
            _unit = pd.DataFrame(_unit)
            _unit.columns = ['unit']
        _unit = pd.DataFrame(_unit)
        _unit.columns = ['unit']
        _new_unit = _unit.unit.str.replace('/'+mon_unit, '')
        _new_unit[_new_unit == ''] = _unit.unit[
            _new_unit == ''].str.replace('/', '')
        _unit.unit = _new_unit

        ext_data[tt].reset_index(level='unit', drop=True, inplace=True)
        ext_dict = extensions.get(ext_name, dict())
        ext_dict.update({table_type: ext_data[tt],
                         'unit': _unit,
                         'name': ext_name})
        extensions.update({ext_name: ext_dict})

    if version[0] == '1':
        year = 2000
    elif version[0] == '2':
        year = 2000
    elif version[0] == '3':
        raise ParserError(
            "This function can not be used to parse EXIOBASE 3")
    else:
        logging.warning("Unknown EXIOBASE version")
        year = None

    return IOSystem(version=version,
                    price='current',
                    year=year,
                    meta=meta_rec,
                    **dict(core_data, **extensions))

Response: Generic parser for EXIOBASE version 1 and 2.


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_MRIO_system(path):
    ispxp = True if re.search('pxp', path, flags=re.IGNORECASE) else False
    isixi = True if re.search('ixi', path, flags=re.IGNORECASE) else False

    if ispxp == isixi:
        system = None
    else:
        system = 'pxp' if ispxp else 'ixi'
    return system

Response: Extract system information from file path.


Instruction:
Explain what the following Python 3 code does
def parse_exiobase1(path):
    path = os.path.abspath(os.path.normpath(str(path)))

    exio_files = get_exiobase_files(path)
    if len(exio_files) == 0:
        raise ParserError("No EXIOBASE files found at {}".format(path))

    system = _get_MRIO_system(path)
    if not system:
        logging.warning("Could not determine system (pxp or ixi)"
                        " set system parameter manually")

    io = generic_exiobase12_parser(exio_files, system=system)
    return io

Response: Parses the EXIOBASE 1 data files and returns a pymrio. IOSystem with the parsed data.


Instruction:
Here you have a function in Python 3, explain what it does
def parse_exiobase2(path, charact=True, popvector='exio2'):
    path = os.path.abspath(os.path.normpath(str(path)))

    exio_files = get_exiobase_files(path)
    if len(exio_files) == 0:
        raise ParserError("No EXIOBASE files found at {}".format(path))

    system = _get_MRIO_system(path)
    if not system:
        logging.warning("Could not determine system (pxp or ixi)"
                        " set system parameter manually")

    io = generic_exiobase12_parser(exio_files, system=system)

    # read the characterisation matrices if available
    # and build one extension with the impacts
    if charact:
        logging.debug('Parse characterisation matrix')
        # dict with correspondence to the extensions
        Qsheets = {'Q_factorinputs': 'factor_inputs',
                   'Q_emission': 'emissions',
                   'Q_materials': 'materials',
                   'Q_resources': 'resources'}

        Q_head_col = dict()
        Q_head_row = dict()
        Q_head_col_rowname = dict()
        Q_head_col_rowunit = dict()
        # Q_head_col_metadata = dict()
        # number of cols containing row headers at the beginning
        Q_head_col['Q_emission'] = 4
        # number of rows containing col headers at the top - this will be
        # skipped
        Q_head_row['Q_emission'] = 3
        # assuming the same classification as in the extensions
        Q_head_col['Q_factorinputs'] = 2
        Q_head_row['Q_factorinputs'] = 2
        Q_head_col['Q_resources'] = 2
        Q_head_row['Q_resources'] = 3
        Q_head_col['Q_materials'] = 2
        Q_head_row['Q_materials'] = 2

        #  column to use as name for the rows
        Q_head_col_rowname['Q_emission'] = 1
        Q_head_col_rowname['Q_factorinputs'] = 0
        Q_head_col_rowname['Q_resources'] = 0
        Q_head_col_rowname['Q_materials'] = 0

        # column to use as unit for the rows which gives also the last column
        # before the data
        Q_head_col_rowunit['Q_emission'] = 3
        Q_head_col_rowunit['Q_factorinputs'] = 1
        Q_head_col_rowunit['Q_resources'] = 1
        Q_head_col_rowunit['Q_materials'] = 1

        if charact is str:
            charac_data = {Qname: pd.read_excel(
                           charact,
                           sheet_name=Qname,
                           skiprows=list(range(0, Q_head_row[Qname])),
                           header=None)
                           for Qname in Qsheets}
        else:
            _content = get_repo_content(path)
            charac_regex = re.compile('(?<!\_)(?<!\.)characterisation.*xlsx')
            charac_files = [ff for ff in _content.filelist if
                            re.search(charac_regex, ff)]
            if len(charac_files) > 1:
                raise ParserError(
                    "Found multiple characcterisation files "
                    "in {} - specify one: {}".format(path, charac_files))
            elif len(charac_files) == 0:
                raise ParserError(
                    "No characcterisation file found "
                    "in {}".format(path))
            else:
                if _content.iszip:
                    with zipfile.ZipFile(path, 'r') as zz:
                        charac_data = {Qname: pd.read_excel(
                                       zz.open(charac_files[0]),
                                       sheet_name=Qname,
                                       skiprows=list(
                                           range(0, Q_head_row[Qname])),
                                       header=None)
                                       for Qname in Qsheets}

                else:
                    charac_data = {Qname: pd.read_excel(
                                   os.path.join(path, charac_files[0]),
                                   sheet_name=Qname,
                                   skiprows=list(range(0, Q_head_row[Qname])),
                                   header=None)
                                   for Qname in Qsheets}

        _unit = dict()
        # temp for the calculated impacts which than
        # get summarized in the 'impact'
        _impact = dict()
        impact = dict()
        for Qname in Qsheets:
            # unfortunately the names in Q_emissions are
            # not completely unique - fix that
            if Qname is 'Q_emission':
                _index = charac_data[Qname][Q_head_col_rowname[Qname]].copy()
                _index.iloc[42] = _index.iloc[42] + ' 2008'
                _index.iloc[43] = _index.iloc[43] + ' 2008'
                _index.iloc[44] = _index.iloc[44] + ' 2010'
                _index.iloc[45] = _index.iloc[45] + ' 2010'
                charac_data[Qname][Q_head_col_rowname[Qname]] = _index

            charac_data[Qname].index = (
                charac_data[Qname][Q_head_col_rowname[Qname]])

            _unit[Qname] = pd.DataFrame(
                    charac_data[Qname].iloc[:, Q_head_col_rowunit[Qname]])
            _unit[Qname].columns = ['unit']
            _unit[Qname].index.name = 'impact'
            charac_data[Qname] = charac_data[Qname].ix[
                :, Q_head_col_rowunit[Qname]+1:]
            charac_data[Qname].index.name = 'impact'

            try:
                _FY = io.__dict__[Qsheets[Qname]].FY.values
            except AttributeError:
                _FY = np.zeros([io.__dict__[Qsheets[Qname]].S.shape[0],
                                io.Y.shape[1]])

            _impact[Qname] = {'S': charac_data[Qname].dot(
                                   io.__dict__[Qsheets[Qname]].S.values),
                              'FY': charac_data[Qname].dot(_FY),
                              'unit': _unit[Qname]
                              }

        impact['S'] = (_impact['Q_factorinputs']['S']
                       .append(_impact['Q_emission']['S'])
                       .append(_impact['Q_materials']['S'])
                       .append(_impact['Q_resources']['S']))
        impact['FY'] = (_impact['Q_factorinputs']['FY']
                        .append(_impact['Q_emission']['FY'])
                        .append(_impact['Q_materials']['FY'])
                        .append(_impact['Q_resources']['FY']))
        impact['S'].columns = io.emissions.S.columns
        impact['FY'].columns = io.emissions.FY.columns
        impact['unit'] = (_impact['Q_factorinputs']['unit']
                          .append(_impact['Q_emission']['unit'])
                          .append(_impact['Q_materials']['unit'])
                          .append(_impact['Q_resources']['unit']))
        impact['name'] = 'impact'
        io.impact = Extension(**impact)

    if popvector is 'exio2':
        logging.debug('Read population vector')
        io.population = pd.read_table(os.path.join(PYMRIO_PATH['exio20'],
                                                   './misc/population.txt'),
                                      index_col=0).astype(float)
    else:
        io.population = popvector

    return io

Response: Parses the EXIOBASE 2. 2 source files and returns a dictionary with the parsed data.


Instruction:
Can you tell what is the following Python 3 function doing
def parse_exiobase3(path):
    io = load_all(path)
    # need to rename the final demand satellite,
    # wrong name in the standard distribution
    try:
        io.satellite.FY = io.satellite.F_hh.copy()
        del io.satellite.F_hh
    except AttributeError:
        pass

    # some ixi in the exiobase 3.4 official distribution
    # have a country name mixup. Clean it here:
    io.rename_regions(
        {'AUS': 'AU',
         'AUT': 'AT',
         'BEL': 'BE',
         'BGR': 'BG',
         'BRA': 'BR',
         'CAN': 'CA',
         'CHE': 'CH',
         'CHN': 'CN',
         'CYP': 'CY',
         'CZE': 'CZ',
         'DEU': 'DE',
         'DNK': 'DK',
         'ESP': 'ES',
         'EST': 'EE',
         'FIN': 'FI',
         'FRA': 'FR',
         'GBR': 'GB',
         'GRC': 'GR',
         'HRV': 'HR',
         'HUN': 'HU',
         'IDN': 'ID',
         'IND': 'IN',
         'IRL': 'IE',
         'ITA': 'IT',
         'JPN': 'JP',
         'KOR': 'KR',
         'LTU': 'LT',
         'LUX': 'LU',
         'LVA': 'LV',
         'MEX': 'MX',
         'MLT': 'MT',
         'NLD': 'NL',
         'NOR': 'NO',
         'POL': 'PL',
         'PRT': 'PT',
         'ROM': 'RO',
         'RUS': 'RU',
         'SVK': 'SK',
         'SVN': 'SI',
         'SWE': 'SE',
         'TUR': 'TR',
         'TWN': 'TW',
         'USA': 'US',
         'ZAF': 'ZA',
         'WWA': 'WA',
         'WWE': 'WE',
         'WWF': 'WF',
         'WWL': 'WL',
         'WWM': 'WM'})

    return io

Response: Parses the public EXIOBASE 3 system and returns a dictionary with the parsed population and characterization data.


Instruction:
How would you explain what the following Python 3 function does
def parse_wiod(path, year=None, names=('isic', 'c_codes'),
               popvector=None):
    """ Parse the wiod source files for the IOSystem

    WIOD provides the MRIO tables in excel - format (xlsx) at
    http://www.wiod.org/new_site/database/wiots.htm (release November 2013).
    To use WIOD in pymrio these (for the year of analysis) must be downloaded.
    The interindustry matrix of these files gets parsed in IOSystem.Z, the
    additional information is included as factor_input extension (value
    added,...)

    The folder with these xslx must than be passed to the WIOD parsing
    function. This folder may contain folders with the extension data. Every
    folder within the wiod root folder will be parsed for extension data and
    will be added to the IOSystem. The WIOD database offers the download of
    the environmental extensions as zip files. These can be read directly by
    the parser. In case a zip file and a folder with the same name are
    available, the data is read from the folder. If the zip files are
    extracted into folder, the folders must have the same name as the
    corresponding zip file (without the 'zip' extension).

    If a WIOD SEA file is present (at the root of path or in a folder named
    'SEA' - only one file!), the labor data of this file gets included in the
    factor_input extension (calculated for the the three skill levels
    available). The monetary data in this file is not added because it is only
    given in national currency.

    Since the "World Input-Output Tables in previous years' prices" are still
    under construction (20141129), no parser for these is provided.

    Some of the meta-parameter of the IOSystem are set automatically based on
    the values given in the first four cells and the name of the WIOD data
    files (base year, version, price, iosystem).
    These can be overwritten afterwards if needed.

    Parameters
    ----------
    path : string or pathlib.Path
        Path to the folder with the WIOD source files. In case that the path
        to a specific file is given, only this will be parsed irrespective of
        the values given in year.
    year : int or str
        Which year in the path should be parsed. The years can be given with
        four or two digits (eg [2012 or 12]). If the given path contains a
        specific file, the value of year will not be used (but inferred from
        the meta data)- otherwise it must be given For the monetary data the
        parser searches for files with 'wiot - two digit year'.
    names : string or tuple, optional
        WIOD provides three different sector/final demand categories naming
        schemes. These can can be specified for the IOSystem. Pass:

            1) 'isic': ISIC rev 3 Codes - available for interindustry flows
               and final demand rows.
            2) 'full': Full names - available for final demand rows and
               final demand columns (categories) and interindustry flows.
            3) 'c_codes' : WIOD specific sector numbers, available for final
               demand rows and columns (categories) and interindustry flows.

        Internally, the parser relies on 1) for the interindustry flows and 3)
        for the final demand categories. This is the default and will also be
        used if just 'isic' gets passed ('c_codes' also replace 'isic' if this
        was passed for final demand categories). To specify different finial
        consumption category names, pass a tuple with (sectors/interindustry
        classification, fd categories), eg ('isic', 'full'). Names are case
        insensitive and passing the first character is sufficient.
    TODO popvector : TO BE IMPLEMENTED (consistent with EXIOBASE)

    Yields
    -------
    IOSystem

    Raises
    ------
    ParserError
        If the WIOD source file are not complete or inconsistent

    """

    # Path manipulation, should work cross platform
    path = os.path.abspath(os.path.normpath(str(path)))

    # wiot start and end
    wiot_ext = '.xlsx'
    wiot_start = 'wiot'

    # determine which wiod file to be parsed
    if not os.path.isdir(path):
        # 1. case - one file specified in path
        if os.path.isfile(path):
            wiot_file = path
        else:
            # just in case the ending was forgotten
            wiot_file = path + wiot_ext
    else:
        # 2. case: directory given-build wiot_file with the value given in year
        if not year:
            raise ParserError('No year specified '
                              '(either specify a specific file '
                              'or a path and year)')
        year_two_digit = str(year)[-2:]
        wiot_file_list = [fl for fl in os.listdir(path)
                          if (fl[:6] == wiot_start + year_two_digit and
                          os.path.splitext(fl)[1] == wiot_ext)]
        if len(wiot_file_list) != 1:
            raise ParserError('Multiple files for a given year or file not '
                              'found (specify a specific file in paramters)')

        wiot_file = os.path.join(path, wiot_file_list[0])

    wiot_file = wiot_file
    root_path = os.path.split(wiot_file)[0]
    if not os.path.exists(wiot_file):
        raise ParserError('WIOD file not found in the specified folder.')

    meta_rec = MRIOMetaData(location=root_path)

    # wiot file structure
    wiot_meta = {
            'col': 0,   # column of the meta information
            'year': 0,  # rest: rows with the data
            'iosystem': 2,
            'unit': 3,
            'end_row': 4,
            }
    wiot_header = {
            # the header indexes are the same for rows after removing the first
            # two lines (wiot_empty_top_rows)
            'code': 0,
            'sector_names': 1,
            'region': 2,
            'c_code': 3,
            }
    wiot_empty_top_rows = [0, 1]

    wiot_marks = {   # special marks
        'last_interindsec': 'c35',     # last sector of the interindustry
        'tot_facinp': ['r60', 'r69'],  # useless totals to remove from factinp
        'total_column': [-1],          # the total column in the whole data
        }

    wiot_sheet = 0   # assume the first one is the one with the data.

    # Wiod has an unfortunate file structure with overlapping metadata and
    # header. In order to deal with that first the full file is read.
    wiot_data = pd.read_excel(wiot_file,
                              sheet_name=wiot_sheet,
                              header=None)

    meta_rec._add_fileio('WIOD data parsed from {}'.format(wiot_file))
    # get meta data
    wiot_year = wiot_data.iloc[wiot_meta['year'], wiot_meta['col']][-4:]
    wiot_iosystem = wiot_data.iloc[
            wiot_meta['iosystem'], wiot_meta['col']].rstrip(')').lstrip('(')
    meta_rec.change_meta('system', wiot_iosystem)
    _wiot_unit = wiot_data.iloc[
            wiot_meta['unit'], wiot_meta['col']].rstrip(')').lstrip('(')

    # remove meta data, empty rows, total column
    wiot_data.iloc[0:wiot_meta['end_row'], wiot_meta['col']] = np.NaN
    wiot_data.drop(wiot_empty_top_rows,
                   axis=0, inplace=True)
    wiot_data.drop(wiot_data.columns[wiot_marks['total_column']],
                   axis=1, inplace=True)
    # at this stage row and column header should have the same size but
    # the index starts now at two - replace/reset to row numbers
    wiot_data.index = range(wiot_data.shape[0])

    # Early years in WIOD tables have a different name for Romania:
    # 'ROM' which should be 'ROU'. The latter is also consistent with
    # the environmental extensions names.
    wiot_data.iloc[wiot_header['region'], :] = wiot_data.iloc[
        wiot_header['region'], :].str.replace('ROM', 'ROU')
    wiot_data.iloc[:, wiot_header['region']] = wiot_data.iloc[
        :, wiot_header['region']].str.replace('ROM', 'ROU')

    # get the end of the interindustry matrix
    _lastZcol = wiot_data[
        wiot_data.iloc[
            :, wiot_header['c_code']] == wiot_marks['last_interindsec']
        ].index[-1]
    _lastZrow = wiot_data[
        wiot_data[wiot_header['c_code']] == wiot_marks['last_interindsec']
        ].index[-1]

    if _lastZcol != _lastZrow:
        raise ParserError(
            'Interindustry matrix not symetric in the WIOD source file')
    else:
        Zshape = (_lastZrow, _lastZcol)

    # separate factor input extension and remove
    # totals in the first and last row
    facinp = wiot_data.iloc[Zshape[0]+1:, :]
    facinp = facinp.drop(
        facinp[facinp[wiot_header['c_code']].isin(
            wiot_marks['tot_facinp'])].index, axis=0
        )

    Z = wiot_data.iloc[:Zshape[0]+1, :Zshape[1]+1].copy()
    Y = wiot_data.iloc[:Zshape[0]+1, Zshape[1]+1:].copy()
    F_fac = facinp.iloc[:, :Zshape[1]+1].copy()
    FY_fac = facinp.iloc[:, Zshape[1]+1:].copy()

    index_wiot_headers = [nr for nr in wiot_header.values()]
    # Save lookup of sectors and codes - to be used at the end of the parser
    # Assuming USA is present in every WIOT year
    wiot_sector_lookup = wiot_data[
        wiot_data[wiot_header['region']] == 'USA'].iloc[
            :, 0:max(index_wiot_headers)+1].applymap(str)
    wiot_sector_lookup.columns = [
        entry[1] for entry in sorted(
            zip(wiot_header.values(), wiot_header.keys()))]
    wiot_sector_lookup.set_index('code', inplace=True, drop=False)
    _Y = Y.T.iloc[:, [
        wiot_header['code'],  # Included to be consistent with  wiot_header
        wiot_header['sector_names'],
        wiot_header['region'],
        wiot_header['c_code'],
        ]]
    wiot_fd_lookup = _Y[_Y.iloc[
        :, wiot_header['region']] == 'USA'].applymap(str)
    wiot_fd_lookup.columns = [
        entry[1] for entry in
        sorted(zip(wiot_header.values(), wiot_header.keys()))]
    wiot_fd_lookup.set_index('c_code', inplace=True, drop=False)
    wiot_fd_lookup.index.name = 'code'

    # set the index/columns, work with code b/c these are also used in the
    # extensions
    Z[wiot_header['code']] = Z[wiot_header['code']].astype(str)
    Z.set_index([wiot_header['region'],
                wiot_header['code']], inplace=True, drop=False)
    Z = Z.iloc[max(index_wiot_headers)+1:, max(index_wiot_headers)+1:]
    Z.index.names = IDX_NAMES['Z_col']
    Z.columns = Z.index

    indexY_col_head = Y.iloc[[wiot_header['region'],
                              wiot_header['c_code']], :]
    Y.columns = pd.MultiIndex.from_arrays(indexY_col_head.values,
                                          names=IDX_NAMES['Y_col2'])
    Y = Y.iloc[max(index_wiot_headers)+1:, :]
    Y.index = Z.index

    F_fac.set_index([wiot_header['sector_names']],
                    inplace=True, drop=False)  # c_code missing, use names
    F_fac.index.names = ['inputtype']
    F_fac = F_fac.iloc[:, max(index_wiot_headers)+1:]
    F_fac.columns = Z.columns
    FY_fac.columns = Y.columns
    FY_fac.index = F_fac.index

    # convert from object to float (was object because mixed float,str)
    Z = Z.astype('float')
    Y = Y.astype('float')
    F_fac = F_fac.astype('float')
    FY_fac = FY_fac.astype('float')

    # save the units
    Z_unit = pd.DataFrame(Z.iloc[:, 0])
    Z_unit.columns = ['unit']
    Z_unit['unit'] = _wiot_unit

    F_fac_unit = pd.DataFrame(F_fac.iloc[:, 0])
    F_fac_unit.columns = ['unit']
    F_fac_unit['unit'] = _wiot_unit

    ll_countries = list(Z.index.get_level_values('region').unique())

    # Finalize the factor inputs extension
    ext = dict()

    ext['factor_inputs'] = {'F': F_fac,
                            'FY': FY_fac,
                            'year': wiot_year,
                            'iosystem': wiot_iosystem,
                            'unit': F_fac_unit,
                            'name': 'factor input',
                            }

    # SEA extension
    _F_sea_data, _F_sea_unit = __get_WIOD_SEA_extension(
                            root_path=root_path, year=year)
    if _F_sea_data is not None:
        # None if no SEA file present
        _FY_sea = pd.DataFrame(index=_F_sea_data.index,
                               columns=FY_fac.columns, data=0)
        _FY_sea = _FY_sea.astype('float')

        ext['SEA'] = {'F': _F_sea_data,
                      'FY': _FY_sea,
                      'year': wiot_year,
                      'iosystem': wiot_iosystem,
                      'unit': _F_sea_unit,
                      'name': 'SEA',
                      }
        meta_rec._add_fileio('SEA file extension parsed from {}'.format(
            root_path))

    # Environmental extensions, names follow the name given
    # in the meta sheet (except for CO2 to get a better description).
    # Units are hardcoded if no consistent place to read them
    # within the files (for all extensions in upper case).
    # The units names must exactly match!
    # Start must identify exactly one folder or a zip file to
    # read the extension.
    # Within the folder, the routine looks for xls files
    # starting with the country code.
    dl_envext_para = {
            'AIR': {'name': 'Air Emission Accounts',
                    'start': 'AIR_',
                    'ext': '.xls',
                    'unit': {
                            'CO2': 'Gg',
                            'CH4': 't',
                            'N2O': 't',
                            'NOx': 't',
                            'SOx': 't',
                            'CO': 't',
                            'NMVOC': 't',
                            'NH3': 't',
                            },
                    },
            'CO2': {'name': 'CO2 emissions - per source',
                    'start': 'CO2_',
                    'ext': '.xls',
                    'unit': {
                        'all': 'Gg'}
                    },

            'EM': {'name': 'Emission relevant energy use',
                   'start': 'EM_',
                   'ext': '.xls',
                   'unit': {
                        'all': 'TJ'}
                   },
            'EU': {'name': 'Gross energy use',
                   'start': 'EU_',
                   'ext': '.xls',
                   'unit': {
                        'all': 'TJ'}
                   },
            'lan': {'name': 'land use',
                    'start': 'lan_',
                    'ext': '.xls',
                    'unit': {
                        'all': None}
                    },
            'mat': {'name': 'material use',
                    'start': 'mat_',
                    'ext': '.xls',
                    'unit': {
                        'all': None}
                    },
            'wat': {'name': 'water use',
                    'start': 'wat_',
                    'ext': '.xls',
                    'unit': {
                        'all': None}
                    },
            }

    _FY_template = pd.DataFrame(columns=FY_fac.columns)
    _ss_FY_pressure_column = 'c37'
    for ik_ext in dl_envext_para:
        _dl_ex = __get_WIOD_env_extension(root_path=root_path,
                                          year=year,
                                          ll_co=ll_countries,
                                          para=dl_envext_para[ik_ext])
        if _dl_ex is not None:
            # None if extension not available
            _FY = _dl_ex['FY']

            _FY.columns = pd.MultiIndex.from_product([
                                _FY.columns, [_ss_FY_pressure_column]])
            _FY = _FY_template.append(_FY)
            _FY.fillna(0, inplace=True)
            _FY.index.names = _dl_ex['F'].index.names
            _FY.columns.names = _FY_template.columns.names
            _FY = _FY[ll_countries]
            _FY = _FY.astype('float')

            ext[ik_ext] = {
                    'F': _dl_ex['F'],
                    'FY': _FY,
                    'year': wiot_year,
                    'iosystem': wiot_iosystem,
                    'unit': _dl_ex['unit'],
                    'name': dl_envext_para[ik_ext]['name'],
                    }
            meta_rec._add_fileio('Extension {} parsed from {}'.format(
                ik_ext, root_path))

    # Build system
    wiod = IOSystem(Z=Z, Y=Y,
                    unit=Z_unit,
                    meta=meta_rec,
                    **ext)

    # Replace sector/final demand category names
    if type(names) is str:
        names = (names, names)
    ll_names = [w[0].lower() for w in names]

    if ll_names[0] == 'c':
        dd_sec_rename = wiot_sector_lookup.c_code.to_dict()
    elif ll_names[0] == 'i':
        dd_sec_rename = wiot_sector_lookup.code.to_dict()
    elif ll_names[0] == 'f':
        dd_sec_rename = wiot_sector_lookup.sector_names.to_dict()
    else:
        dd_sec_rename = wiot_sector_lookup.code.to_dict()
        warnings.warn('Parameter for names not understood - '
                      'used ISIC codes as sector names')

    if ll_names[1] == 'c':
        dd_fd_rename = wiot_fd_lookup.c_code.to_dict()
    elif ll_names[1] == 'i':
        dd_fd_rename = wiot_fd_lookup.c_code.to_dict()
    elif ll_names[1] == 'f':
        dd_fd_rename = wiot_fd_lookup.sector_names.to_dict()
    else:
        warnings.warn('Parameter for names not understood - '
                      'used c_codes as final demand category names')

    wiod.Z.rename(columns=dd_sec_rename, index=dd_sec_rename, inplace=True)
    wiod.Y.rename(columns=dd_fd_rename, index=dd_sec_rename, inplace=True)
    for ext in wiod.get_extensions(data=True):
        ext.F.rename(columns=dd_sec_rename, inplace=True)
        ext.FY.rename(columns=dd_fd_rename, inplace=True)

    return wiod

Response: Parses the WIOD source files for the IOSystem and returns a tuple of the base year version price and popvector.


Instruction:
Write a Python 3 script to
parse the WIOD environmental extension and returns the dictionary of the extension data for the given year.

Response:
def __get_WIOD_env_extension(root_path, year, ll_co, para):
    """ Parses the wiod environmental extension

    Extension can either be given as original .zip files or as extracted
    data in a folder with the same name as the corresponding zip file (with-
    out the extension).

    This function is based on the structure of the extensions from _may12.

    Note
    ----
    The function deletes 'secQ' which is not present in the economic tables.

    Parameters
    ----------
    root_path : string
        Path to the WIOD data or the path with the
        extension data folder or zip file.
    year : str or int
        Year to return for the extension = valid sheetname for the xls file.
    ll_co : list like
        List of countries in WIOD - used for finding and matching
        extension data in the given folder.
    para : dict
        Defining the parameters for reading the extension.

    Returns
    -------
    dict with keys
        F : pd.DataFrame with index 'stressor' and columns 'region', 'sector'
        FY : pd.Dataframe with index 'stressor' and column 'region'
            This data is for household stressors - must be applied to the right
            final demand column afterwards.
        unit : pd.DataFrame with index 'stressor' and column 'unit'

    """

    ll_root_content = [ff for ff in os.listdir(root_path) if
                       ff.startswith(para['start'])]
    if len(ll_root_content) < 1:
        warnings.warn(
            'Extension data for {} not found - '
            'Extension not included'.format(para['start']), ParserWarning)
        return None

    elif len(ll_root_content) > 1:
        raise ParserError(
            'Several raw data for extension'
            '{} available - clean extension folder.'.format(para['start']))

    pf_env = os.path.join(root_path, ll_root_content[0])

    if pf_env.endswith('.zip'):
        rf_zip = zipfile.ZipFile(pf_env)
        ll_env_content = [ff for ff in rf_zip.namelist() if
                          ff.endswith(para['ext'])]
    else:
        ll_env_content = [ff for ff in os.listdir(pf_env) if
                          ff.endswith(para['ext'])]

    dl_env = dict()
    dl_env_hh = dict()
    for co in ll_co:
        ll_pff_read = [ff for ff in ll_env_content if
                       ff.endswith(para['ext']) and
                       (ff.startswith(co.upper()) or
                        ff.startswith(co.lower()))]

        if len(ll_pff_read) < 1:
            raise ParserError('Country data not complete for Extension '
                              '{} - missing {}.'.format(para['start'], co))

        elif len(ll_pff_read) > 1:
            raise ParserError('Multiple country data for Extension '
                              '{} - country {}.'.format(para['start'], co))

        pff_read = ll_pff_read[0]

        if pf_env.endswith('.zip'):
            ff_excel = pd.ExcelFile(rf_zip.open(pff_read))
        else:
            ff_excel = pd.ExcelFile(os.path.join(pf_env, pff_read))
        if str(year) in ff_excel.sheet_names:
            df_env = ff_excel.parse(sheet_name=str(year),
                                    index_col=None,
                                    header=0
                                    )
        else:
            warnings.warn('Extension {} does not include'
                          'data for the year {} - '
                          'Extension not included'.format(para['start'], year),
                          ParserWarning)
            return None

        if not df_env.index.is_numeric():
            # upper case letter extensions gets parsed with multiindex, not
            # quite sure why...
            df_env.reset_index(inplace=True)

        # unit can be taken from the first cell in the excel sheet
        if df_env.columns[0] != 'level_0':
            para['unit']['all'] = df_env.columns[0]

        # two clean up cases - can be identified by lower/upper case extension
        # description
        if para['start'].islower():
            pass
        elif para['start'].isupper():
            df_env = df_env.iloc[:, 1:]
        else:
            raise ParserError('Format of extension not given.')

        df_env.dropna(axis=0, how='all', inplace=True)
        df_env = df_env[df_env.iloc[:, 0] != 'total']
        df_env = df_env[df_env.iloc[:, 0] != 'secTOT']
        df_env = df_env[df_env.iloc[:, 0] != 'secQ']
        df_env.iloc[:, 0].astype(str, inplace=True)
        df_env.iloc[:, 0].replace(to_replace='sec',
                                  value='',
                                  regex=True,
                                  inplace=True)

        df_env.set_index([df_env.columns[0]], inplace=True)
        df_env.index.names = ['sector']
        df_env = df_env.T

        ikc_hh = 'FC_HH'
        dl_env_hh[co] = df_env[ikc_hh]
        del df_env[ikc_hh]
        dl_env[co] = df_env

    df_F = pd.concat(dl_env, axis=1)[ll_co]
    df_FY = pd.concat(dl_env_hh, axis=1)[ll_co]
    df_F.fillna(0, inplace=True)
    df_FY.fillna(0, inplace=True)

    df_F.columns.names = IDX_NAMES['F_col']
    df_F.index.names = IDX_NAMES['F_row_single']

    df_FY.columns.names = IDX_NAMES['Y_col1']
    df_FY.index.names = IDX_NAMES['F_row_single']

    # build the unit df
    df_unit = pd.DataFrame(index=df_F.index, columns=['unit'])
    _ss_unit = para['unit'].get('all', 'undef')
    for ikr in df_unit.index:
        df_unit.ix[ikr, 'unit'] = para['unit'].get(ikr, _ss_unit)

    df_unit.columns.names = ['unit']
    df_unit.index.names = ['stressor']

    if pf_env.endswith('.zip'):
        rf_zip.close()

    return {'F': df_F,
            'FY': df_FY,
            'unit': df_unit
            }


Instruction:
Create a Python 3 function for
parsing the Eora26 database file and returns a dictionary of statistical disrecpancy columns.

Response:
def parse_eora26(path, year=None, price='bp', country_names='eora'):
    """ Parse the Eora26 database

    Note
    ----

    This parser deletes the statistical disrecpancy columns from
    the parsed Eora system (reports the amount of loss in the
    meta recors).

    Eora does not provide any information on the unit of the
    monetary values. Based on personal communication the unit
    is set to Mill USD manually.

    Parameters
    ----------

    path : string or pathlib.Path
       Path to the eora raw storage folder or a specific eora zip file to
       parse.  There are several options to specify the data for parsing:

       1) Pass the name of eora zip file. In this case the parameters 'year'
          and 'price' will not be used
       2) Pass a folder which eiter contains eora zip files or unpacked eora
          data In that case, a year must be given
       3) Pass a folder which contains subfolders in the format 'YYYY', e.g.
          '1998' This subfolder can either contain an Eora zip file or an
          unpacked Eora system

    year : int or str
        4 digit year spec. This will not be used if a zip file
        is specified in 'path'

    price : str, optional
        'bp' or 'pp'

    country_names: str, optional
        Which country names to use:
        'eora' = Eora flavoured ISO 3 varian
        'full' = Full country names as provided by Eora
        Passing the first letter suffice.

    """
    path = os.path.abspath(os.path.normpath(str(path)))

    if country_names[0].lower() == 'e':
        country_names = 'eora'
    elif country_names[0].lower() == 'f':
        country_names = 'full'
    else:
        raise ParserError('Parameter country_names must be eora or full')

    row_name = 'ROW'
    eora_zip_ext = '.zip'
    is_zip = False

    # determine which eora file to be parsed
    if os.path.splitext(path)[1] == eora_zip_ext:
        # case direct pass of eora zipfile
        year = re.search(r'\d\d\d\d',
                         os.path.basename(path)).group(0)
        price = re.search(r'bp|pp',
                          os.path.basename(path)).group(0)
        eora_loc = path
        root_path = os.path.split(path)[0]
        is_zip = True
    else:
        root_path = path
        if str(year) in os.listdir(path):
            path = os.path.join(path, str(year))

        eora_file_list = [fl for fl in os.listdir(path)
                          if os.path.splitext(fl)[1] == eora_zip_ext and
                          str(year) in fl and
                          str(price) in fl
                          ]

        if len(eora_file_list) > 1:
            raise ParserError('Multiple files for a given year '
                              'found (specify a specific file in paramters)')
        elif len(eora_file_list) == 1:
            eora_loc = os.path.join(path, eora_file_list[0])
            is_zip = True
        else:
            # Just a path was given, no zip file found,
            # continue with only the path information - assumed an
            # unpacked zip file
            eora_loc = path
            is_zip = False

    meta_rec = MRIOMetaData(location=root_path)

    # Eora file specs
    eora_sep = '\t'
    ZY_col = namedtuple('ZY', 'full eora system name')(0, 1, 2, 3)

    eora_files = {
        'Z': 'Eora26_{year}_{price}_T.txt'.format(
            year=str(year), price=price),
        'Q': 'Eora26_{year}_{price}_Q.txt'.format(
            year=str(year), price=price),
        'QY': 'Eora26_{year}_{price}_QY.txt'.format(
            year=str(year), price=price),
        'VA': 'Eora26_{year}_{price}_VA.txt'.format(
            year=str(year), price=price),
        'Y': 'Eora26_{year}_{price}_FD.txt'.format(
            year=str(year), price=price),
        'labels_Z': 'labels_T.txt',
        'labels_Y': 'labels_FD.txt',
        'labels_Q': 'labels_Q.txt',
        'labels_VA': 'labels_VA.txt',
        }

    header = namedtuple('header', 'index columns index_names, column_names')

    eora_header_spec = {
        'Z': header(index='labels_Z',
                    columns='labels_Z',
                    index_names=IDX_NAMES['Z_row'],
                    column_names=IDX_NAMES['Z_col'],
                    ),
        'Q': header(index='labels_Q',
                    columns='labels_Z',
                    index_names=IDX_NAMES['F_row_src'],
                    column_names=IDX_NAMES['F_col']),
        'QY': header(index='labels_Q',
                     columns='labels_Y',
                     index_names=IDX_NAMES['F_row_src'],
                     column_names=IDX_NAMES['Y_col2'],
                     ),
        'VA': header(index='labels_VA',
                     columns='labels_Z',
                     index_names=IDX_NAMES['VA_row_unit_cat'],
                     column_names=IDX_NAMES['F_col']
                     ),
        'Y': header(index='labels_Z',
                    columns='labels_Y',
                    index_names=IDX_NAMES['Y_row'],
                    column_names=IDX_NAMES['Y_col2'],
                    ),
        }

    if is_zip:
        zip_file = zipfile.ZipFile(eora_loc)
        eora_data = {
            key: pd.read_table(
                zip_file.open(filename),
                sep=eora_sep,
                header=None
                ) for
            key, filename in eora_files.items()}
        zip_file.close()
    else:
        eora_data = {
            key: pd.read_table(
                os.path.join(eora_loc, filename),
                sep=eora_sep,
                header=None
                ) for
            key, filename in eora_files.items()}
    meta_rec._add_fileio(
        'Eora26 for {year}-{price} data parsed from {loc}'.format(
            year=year, price=price, loc=eora_loc))

    eora_data['labels_Z'] = eora_data[
        'labels_Z'].loc[:, [getattr(ZY_col, country_names), ZY_col.name]]
    eora_data['labels_Y'] = eora_data[
        'labels_Y'].loc[:, [getattr(ZY_col, country_names), ZY_col.name]]
    eora_data['labels_VA'] = eora_data[
        'labels_VA'].iloc[:, :len(eora_header_spec['VA'].column_names)]
    labQ = eora_data[
        'labels_Q'].iloc[:, :len(eora_header_spec['Q'].column_names)]
    labQ.columns = IDX_NAMES['F_row_src']
    Q_unit = labQ['stressor'].str.extract(r'\((.*)\)', expand=False)
    Q_unit.columns = IDX_NAMES['unit']

    labQ['stressor'] = labQ['stressor'].str.replace(r'\s\((.*)\)', '')
    eora_data['labels_Q'] = labQ

    for key in eora_header_spec.keys():
        eora_data[key].columns = (
            eora_data[eora_header_spec[key].columns].set_index(list(
                eora_data[eora_header_spec[key].columns])).index)
        eora_data[key].columns.names = eora_header_spec[key].column_names
        eora_data[key].index = (
            eora_data[eora_header_spec[key].index].set_index(list(
                eora_data[eora_header_spec[key].index])).index)
        eora_data[key].index.names = eora_header_spec[key].index_names

        try:
            meta_rec._add_modify(
                'Remove Rest of the World ({name}) '
                'row from {table} - loosing {amount}'.format(
                    name=row_name,
                    table=key,
                    amount=eora_data[key].loc[:, row_name].sum().values[0]))
            eora_data[key].drop(row_name, axis=1, inplace=True)
        except KeyError:
            pass

        try:
            meta_rec._add_modify(
                'Remove Rest of the World ({name}) column '
                'from {table} - loosing {amount}'.format(
                    name=row_name,
                    table=key,
                    amount=eora_data[key].loc[row_name, :].sum().values[0]))
            eora_data[key].drop(row_name, axis=0, inplace=True)
        except KeyError:
            pass

    Q_unit.index = eora_data['Q'].index

    meta_rec.note('Set Eora moneatry units to Mill USD manually')
    Z_unit = pd.DataFrame(data=['Mill USD'] * len(eora_data['Z'].index),
                          index=eora_data['Z'].index,
                          columns=['unit'])
    VA_unit = pd.DataFrame(data=['Mill USD'] * len(eora_data['VA'].index),
                           index=eora_data['VA'].index,
                           columns=['unit'])

    eora = IOSystem(
        Z=eora_data['Z'],
        Y=eora_data['Y'],
        unit=Z_unit,
        Q={
            'name': 'Q',
            'unit': Q_unit,
            'F': eora_data['Q'],
            'FY': eora_data['QY']
            },
        VA={
            'name': 'VA',
            'F': eora_data['VA'],
            'unit': VA_unit,
             },
        meta=meta_rec)

    return eora


Instruction:
Can you generate the documentation for the following Python 3 function
def _add_history(self, entry_type, entry):
        meta_string = "{time} - {etype} -  {entry}".format(
            time=self._time(),
            etype=entry_type.upper(),
            entry=entry)

        self._content['history'].insert(0, meta_string)
        self.logger(meta_string)

Response: Generic method to add entry as entry_type to the history


Instruction:
Can you implement a function in Python 3 that
changes the meta data entry to a new value.

Response:
def change_meta(self, para, new_value, log=True):
        """ Changes the meta data

        This function does nothing if None is passed as new_value.
        To set a certain value to None pass the str 'None'

        Parameters
        ----------
        para: str
            Meta data entry to change

        new_value: str
            New value

        log: boolean, optional
            If True (default) records the meta data change
            in the history

        """
        if not new_value:
            return
        para = para.lower()
        if para == 'history':
            raise ValueError(
                'History can only be extended - use method "note"')
        old_value = self._content.get(para, None)
        if new_value == old_value:
            return
        self._content[para] = new_value
        if old_value and log:
            self._add_history(entry_type="METADATA_CHANGE",
                              entry='Changed parameter "{para}" '
                                    'from "{old}" to "{new}"'.format(
                                        para=para,
                                        old=old_value,
                                        new=new_value))


Instruction:
Given the following Python 3 function, write the documentation
def _read_content(self):
        if self._path_in_arc:
            with zipfile.ZipFile(file=str(self._metadata_file)) as zf:
                self._content = json.loads(
                    zf.read(self._path_in_arc).decode('utf-8'),
                    object_pairs_hook=OrderedDict)
        else:
            with self._metadata_file.open('r') as mdf:
                self._content = json.load(mdf,
                                          object_pairs_hook=OrderedDict)

Response: Reads the content of the object from the file.


Instruction:
Can you create a Python 3 function that
saves the current status of the metadata to the given location.

Response:
def save(self, location=None):
        """ Saves the current status of the metadata

        This saves the metadata at the location of the previously loaded
        metadata or at the file/path given in location.

        Specify a location if the metadata should be stored in a different
        location or was never stored before. Subsequent saves will use the
        location set here.

        Parameters
        ----------
        location: str, optional
            Path or file for saving the metadata.
            This can be the full file path or just the storage folder.
            In the latter case, the filename defined in
            DEFAULT_FILE_NAMES['metadata'] (currently 'metadata.json') is
            assumed.
        """
        if location:
            location = Path(location)
            if os.path.splitext(str(location))[1] == '':
                self._metadata_file = location / DEFAULT_FILE_NAMES['metadata']
            else:
                self._metadata_file = location

        if self._metadata_file:
            with self._metadata_file.open(mode='w') as mdf:
                json.dump(self._content, mdf, indent=4)
        else:
            logging.error("No metadata file given for storing the file")


Instruction:
Can you write a function in Python 3 where it
calculates the industry output x from the Z and Y matrix.

Response:
def calc_x(Z, Y):
    """ Calculate the industry output x from the Z and Y matrix

    Parameters
    ----------
    Z : pandas.DataFrame or numpy.array
        Symmetric input output table (flows)
    Y : pandas.DataFrame or numpy.array
        final demand with categories (1.order) for each country (2.order)

    Returns
    -------
    pandas.DataFrame or numpy.array
        Industry output x as column vector
        The type is determined by the type of Z. If DataFrame index as Z

    """
    x = np.reshape(np.sum(np.hstack((Z, Y)), 1), (-1, 1))
    if type(Z) is pd.DataFrame:
        x = pd.DataFrame(x, index=Z.index, columns=['indout'])
    if type(x) is pd.Series:
        x = pd.DataFrame(x)
    if type(x) is pd.DataFrame:
        x.columns = ['indout']
    return x


Instruction:
Can you implement a function in Python 3 that
calculates the industry output x from a symmetric output Leontief table L and a column vector y

Response:
def calc_x_from_L(L, y):
    """ Calculate the industry output x from L and a y vector

    Parameters
    ----------
    L : pandas.DataFrame or numpy.array
        Symmetric input output Leontief table
    y : pandas.DataFrame or numpy.array
        a column vector of the total final demand

    Returns
    -------
    pandas.DataFrame or numpy.array
        Industry output x as column vector
        The type is determined by the type of L. If DataFrame index as L

    """
    x = L.dot(y)
    if type(x) is pd.Series:
        x = pd.DataFrame(x)
    if type(x) is pd.DataFrame:
        x.columns = ['indout']
    return x


Instruction:
Here you have a function in Python 3, explain what it does
def calc_Z(A, x):
    if (type(x) is pd.DataFrame) or (type(x) is pd.Series):
        x = x.values
    x = x.reshape((1, -1))   # use numpy broadcasting - much faster
    # (but has to ensure that x is a row vector)
    # old mathematical form:
    # return A.dot(np.diagflat(x))
    if type(A) is pd.DataFrame:
        return pd.DataFrame(A.values * x, index=A.index, columns=A.columns)
    else:
        return A*x

Response: Calculates the Z matrix from A and x


Instruction:
Implement a Python 3 function for
calculating the A matrix from Z and x.

Response:
def calc_A(Z, x):
    """ Calculate the A matrix (coefficients) from Z and x

    Parameters
    ----------
    Z : pandas.DataFrame or numpy.array
        Symmetric input output table (flows)
    x : pandas.DataFrame or numpy.array
        Industry output column vector

    Returns
    -------
    pandas.DataFrame or numpy.array
        Symmetric input output table (coefficients) A
        The type is determined by the type of Z.
        If DataFrame index/columns as Z

    """
    if (type(x) is pd.DataFrame) or (type(x) is pd.Series):
        x = x.values
    if (type(x) is not np.ndarray) and (x == 0):
        recix = 0
    else:
        with warnings.catch_warnings():
            # catch the divide by zero warning
            # we deal wit that by setting to 0 afterwards
            warnings.simplefilter('ignore')
            recix = 1/x
        recix[recix == np.inf] = 0
        recix = recix.reshape((1, -1))
    # use numpy broadcasting - factor ten faster
    # Mathematical form - slow
    # return Z.dot(np.diagflat(recix))
    if type(Z) is pd.DataFrame:
        return pd.DataFrame(Z.values * recix, index=Z.index, columns=Z.columns)
    else:
        return Z*recix


Instruction:
How would you explain what the following Python 3 function does
def calc_L(A):
    I = np.eye(A.shape[0])   # noqa
    if type(A) is pd.DataFrame:
        return pd.DataFrame(np.linalg.inv(I-A),
                            index=A.index, columns=A.columns)
    else:
        return np.linalg.inv(I-A)

Response: Calculates the Leontief L from A


Instruction:
Write a Python 3 script for
calculating Multipliers based on footprints.

Response:
def recalc_M(S, D_cba, Y, nr_sectors):
    """ Calculate Multipliers based on footprints.

    Parameters
    ----------
    D_cba : pandas.DataFrame or numpy array
        Footprint per sector and country
    Y : pandas.DataFrame or numpy array
        Final demand: aggregated across categories or just one category, one
        column per country. This will be diagonalized per country block.
        The diagonolized form must be invertable for this method to work.
    nr_sectors : int
        Number of sectors in the MRIO

    Returns
    -------

    pandas.DataFrame or numpy.array
        Multipliers M
        The type is determined by the type of D_cba.
        If DataFrame index/columns as D_cba

    """

    Y_diag = ioutil.diagonalize_blocks(Y.values, blocksize=nr_sectors)
    Y_inv = np.linalg.inv(Y_diag)
    M = D_cba.dot(Y_inv)
    if type(D_cba) is pd.DataFrame:
        M.columns = D_cba.columns
        M.index = D_cba.index

    return M


Instruction:
Write a Python 3 script for
calculating the total cba and pba based accounts imp and exp accounts for a given set of sectors in the MRIO.

Response:
def calc_accounts(S, L, Y, nr_sectors):
    """ Calculate sector specific cba and pba based accounts, imp and exp accounts

    The total industry output x for the calculation
    is recalculated from L and y

    Parameters
    ----------
    L : pandas.DataFrame
        Leontief input output table L
    S : pandas.DataFrame
        Direct impact coefficients
    Y : pandas.DataFrame
        Final demand: aggregated across categories or just one category, one
        column per country
    nr_sectors : int
        Number of sectors in the MRIO

    Returns
    -------
    Tuple
        (D_cba, D_pba, D_imp, D_exp)

        Format: D_row x L_col (=nr_countries*nr_sectors)

        - D_cba        Footprint per sector and country
        - D_pba      Total factur use per sector and country
        - D_imp       Total global factor use to satisfy total final demand in
                      the country per sector
        - D_exp       Total factor use in one country to satisfy final demand
                      in all other countries (per sector)
    """
    # diagonalize each sector block per country
    # this results in a disaggregated y with final demand per country per
    # sector in one column
    Y_diag = ioutil.diagonalize_blocks(Y.values, blocksize=nr_sectors)
    x_diag = L.dot(Y_diag)
    x_tot = x_diag.values.sum(1)
    del Y_diag

    D_cba = pd.DataFrame(S.values.dot(x_diag),
                         index=S.index,
                         columns=S.columns)
    # D_pba = S.dot(np.diagflat(x_tot))
    # faster broadcasted calculation:
    D_pba = pd.DataFrame(S.values*x_tot.reshape((1, -1)),
                         index=S.index,
                         columns=S.columns)

    # for the traded accounts set the domestic industry output to zero
    dom_block = np.zeros((nr_sectors, nr_sectors))
    x_trade = ioutil.set_block(x_diag.values, dom_block)
    D_imp = pd.DataFrame(S.values.dot(x_trade),
                         index=S.index,
                         columns=S.columns)

    x_exp = x_trade.sum(1)
    # D_exp = S.dot(np.diagflat(x_exp))
    # faster broadcasted version:
    D_exp = pd.DataFrame(S.values * x_exp.reshape((1, -1)),
                         index=S.index,
                         columns=S.columns)

    return (D_cba, D_pba, D_imp, D_exp)


Instruction:
Can you write a function in Python 3 where it
gets the url datafiles from the database

Response:
def _get_url_datafiles(url_db_view, url_db_content,
                       mrio_regex, access_cookie=None):
    """ Urls of mrio files by parsing url content for mrio_regex

    Parameters
    ----------

    url_db_view: url str
        Url which shows the list of mrios in the db

    url_db_content: url str
        Url which needs to be appended before the url parsed from the
        url_db_view to get a valid download link

    mrio_regex: regex str
        Regex to parse the mrio datafile from url_db_view

    access_cookie: dict, optional
        If needed, cookie to access the database

    Returns
    -------
    Named tuple:
    .raw_text: content of url_db_view for later use
    .data_urls: list of url

    """
    # Use post here - NB: get could be necessary for some other pages
    # but currently works for wiod and eora
    returnvalue = namedtuple('url_content',
                             ['raw_text', 'data_urls'])
    url_text = requests.post(url_db_view, cookies=access_cookie).text
    data_urls = [url_db_content + ff
                 for ff in re.findall(mrio_regex, url_text)]
    return returnvalue(raw_text=url_text, data_urls=data_urls)


Instruction:
How would you implement a function in Python 3 that
downloads a list of urls to the storage_folder

Response:
def _download_urls(url_list, storage_folder, overwrite_existing,
                   meta_handler, access_cookie=None):
    """ Save url from url_list to storage_folder

    Parameters
    ----------
    url_list: list of str
        Valid url to download

    storage_folder: str, valid path
        Location to store the download, folder will be created if
        not existing. If the file is already present in the folder,
        the download depends on the setting in 'overwrite_existing'.

    overwrite_existing: boolean, optional
        If False, skip download of file already existing in
        the storage folder (default). Set to True to replace
        files.

    meta_handler: instance of MRIOMetaData

    Returns
    -------

    The meta_handler is passed back

    """
    for url in url_list:
        filename = os.path.basename(url)
        if not overwrite_existing and filename in os.listdir(storage_folder):
            continue
        storage_file = os.path.join(storage_folder, filename)

        # Using requests here - tried with aiohttp but was actually slower
        # Also don’t use shutil.copyfileobj - corrupts zips from Eora
        req = requests.post(url, stream=True, cookies=access_cookie)
        with open(storage_file, 'wb') as lf:
            for chunk in req.iter_content(1024*5):
                lf.write(chunk)

        meta_handler._add_fileio('Downloaded {} to {}'.format(url, filename))
        meta_handler.save()

    return meta_handler


Instruction:
Implement a Python 3 function for
downloading the 2013 wiod release and store it in the given storage folder.

Response:
def download_wiod2013(storage_folder, years=None, overwrite_existing=False,
                      satellite_urls=WIOD_CONFIG['satellite_urls']):
    """ Downloads the 2013 wiod release

    Note
    ----
    Currently, pymrio only works with the 2013 release of the wiod tables. The
    more recent 2016 release so far (October 2017) lacks the environmental and
    social extensions.

    Parameters
    ----------
    storage_folder: str, valid path
        Location to store the download, folder will be created if
        not existing. If the file is already present in the folder,
        the download of the specific file will be skipped.

    years: list of int or str, optional
        If years is given only downloads the specific years. This
        only applies to the IO tables because extensions are stored
        by country and not per year.
        The years can be given in 2 or 4 digits.

    overwrite_existing: boolean, optional
        If False, skip download of file already existing in
        the storage folder (default). Set to True to replace
        files.

    satellite_urls : list of str (urls), optional
        Which satellite accounts to download.  Default: satellite urls defined
        in WIOD_CONFIG - list of all available urls Remove items from this list
        to only download a subset of extensions

    """

    try:
        os.makedirs(storage_folder)
    except FileExistsError:
        pass

    if type(years) is int or type(years) is str:
        years = [years]
    years = years if years else range(1995, 2012)
    years = [str(yy).zfill(2)[-2:] for yy in years]

    wiod_web_content = _get_url_datafiles(
        url_db_view=WIOD_CONFIG['url_db_view'],
        url_db_content=WIOD_CONFIG['url_db_content'],
        mrio_regex='protected.*?wiot\d\d.*?xlsx')

    restricted_wiod_io_urls = [url for url in wiod_web_content.data_urls if
                               re.search(r"(wiot)(\d\d)",
                                         os.path.basename(url)).group(2)
                               in years]

    meta = MRIOMetaData(location=storage_folder,
                        description='WIOD metadata file for pymrio',
                        name='WIOD',
                        system='ixi',
                        version='data13')

    meta = _download_urls(url_list=restricted_wiod_io_urls + satellite_urls,
                          storage_folder=storage_folder,
                          overwrite_existing=overwrite_existing,
                          meta_handler=meta)

    meta.save()
    return meta


Instruction:
Implement a Python 3 function for
getting a timestamp of length in string

Response:
def get_timestamp(length):
    """Get a timestamp of `length` in string"""
    s = '%.6f' % time.time()
    whole, frac = map(int, s.split('.'))
    res = '%d%d' % (whole, frac)
    return res[:length]


Instruction:
Make a summary of the following Python 3 code
def monkey_patch():
    ozmq = __import__('zmq')
    ozmq.Socket = zmq.Socket
    ozmq.Context = zmq.Context
    ozmq.Poller = zmq.Poller
    ioloop = __import__('zmq.eventloop.ioloop')
    ioloop.Poller = zmq.Poller

Response: Monkey patches zmq. Context and zmq. Socket with zmq. Poller and zmq. EventLoop


Instruction:
Can you tell what is the following Python 3 function doing
def concate_extension(*extensions, name):
    if type(extensions[0]) is tuple or type(extensions[0]) is list:
        extensions = extensions[0]

    # check if fd extensions is present in one of the given extensions
    FY_present = False
    SY_present = False
    SFY_columns = None
    for ext in extensions:
        if 'FY' in ext.get_DataFrame(data=False):
            FY_present = True
            SFY_columns = ext.FY.columns
        if 'SY' in ext.get_DataFrame(data=False):
            SY_present = True
            SFY_columns = ext.SY.columns

    # get the intersection of the available dataframes
    set_dfs = [set(ext.get_DataFrame(data=False)) for ext in extensions]
    df_dict = {key: None for key in set.intersection(*set_dfs)}
    if FY_present:
        df_dict['FY'] = None
    if SY_present:
        df_dict['SY'] = None
    empty_df_dict = df_dict.copy()
    attr_dict = {}

    # get data from each extension
    first_run = True
    for ext in extensions:
        # get corresponding attributes of all extensions
        for key in ext.__dict__:
            if type(ext.__dict__[key]) is not pd.DataFrame:
                if attr_dict.get(key, -99) == -99:
                    attr_dict[key] = ext.__dict__[key]
                elif attr_dict[key] == ext.__dict__[key]:
                    continue
                else:
                    attr_dict[key] = None

        # get DataFrame data
        cur_dict = empty_df_dict.copy()

        for df in cur_dict:
            cur_dict[df] = getattr(ext, df)

        # add zero final demand extension if final demand extension present in
        # one extension
        if FY_present:
            # doesn't work with getattr b/c FY can be present as attribute but
            # not as DataFrame
            if 'FY' in ext.get_DataFrame(data=False):
                cur_dict['FY'] = getattr(ext, 'FY')
            else:
                cur_dict['FY'] = pd.DataFrame(data=0,
                                              index=ext.get_index(),
                                              columns=SFY_columns)
        if SY_present:
            # doesn't work with getattr b/c SY can be present as attribute but
            # not as DataFrame
            if 'SY' in ext.get_DataFrame(data=False):
                cur_dict['SY'] = getattr(ext, 'SY')
            else:
                cur_dict['SY'] = pd.DataFrame(data=0,
                                              index=ext.get_index(),
                                              columns=SFY_columns)

        # append all df data
        for key in cur_dict:
            if not first_run:
                if cur_dict[key].index.names != df_dict[key].index.names:
                    cur_ind_names = list(cur_dict[key].index.names)
                    df_ind_names = list(df_dict[key].index.names)
                    cur_ind_names[0] = 'indicator'
                    df_ind_names[0] = cur_ind_names[0]
                    cur_dict[key].index.set_names(cur_ind_names,
                                                  inplace=True)
                    df_dict[key].index.set_names(df_ind_names,
                                                 inplace=True)

                    for ind in cur_ind_names:
                        if ind not in df_ind_names:
                            df_dict[key] = (df_dict[key].
                                            set_index(pd.DataFrame(
                                                data=None,
                                                index=df_dict[key].index,
                                                columns=[ind])[ind],
                                                append=True))
                    for ind in df_ind_names:
                        if ind not in cur_ind_names:
                            cur_dict[key] = (cur_dict[key].set_index(
                                                pd.DataFrame(
                                                    data=None,
                                                    index=cur_dict[key].index,
                                                    columns=[ind])
                                                [ind], append=True))

            df_dict[key] = pd.concat([df_dict[key], cur_dict[key]])

        first_run = False

        all_dict = dict(list(attr_dict.items()) + list(df_dict.items()))
        all_dict['name'] = name

    return Extension(**all_dict)

Response: Concatenate a list of extensions into a single list of new extensions.


Instruction:
Make a summary of the following Python 3 code
def reset_to_flows(self, force=False, _meta=None):
        # Development note: The attributes which should be removed are
        # defined in self.__non_agg_attributes__
        strwarn = None
        for df in self.__basic__:
            if (getattr(self, df)) is None:
                if force:
                    strwarn = ("Reset system warning - Recalculation after "
                               "reset not possible "
                               "because {} missing".format(df))
                    warnings.warn(strwarn, ResetWarning)

                else:
                    raise ResetError("To few tables to recalculate the "
                                     "system after reset ({} missing) "
                                     "- reset can be forced by passing "
                                     "'force=True')".format(df))

        if _meta:
            _meta._add_modify("Reset to absolute flows")
            if strwarn:
                _meta._add_modify(strwarn)

        [setattr(self, key, None) for key in self.__non_agg_attributes__]
        return self

Response: Resets the internal object to absolute values.


Instruction:
Write a Python 3 script for
resetting the coefficient attributes of the current object to the coefficient values of the current object.

Response:
def reset_to_coefficients(self):
        """ Keeps only the coefficient.

        This can be used to recalculate the IO tables for a new finald demand.

        Note
        -----

        The system can not be reconstructed after this steps
        because all absolute data is removed. Save the Y data in case
        a reconstruction might be necessary.

        """
        # Development note: The coefficient attributes are
        # defined in self.__coefficients__
        [setattr(self, key, None)
         for key in self.get_DataFrame(
             data=False,
             with_unit=False,
             with_population=False)
         if key not in self.__coefficients__]
        return self


Instruction:
Can you implement a function in Python 3 that
returns a deep copy of the system

Response:
def copy(self, new_name=None):
        """ Returns a deep copy of the system

        Parameters
        -----------

        new_name: str, optional
            Set a new meta name parameter.
            Default: <old_name>_copy
        """
        _tmp = copy.deepcopy(self)
        if not new_name:
            new_name = self.name + '_copy'
        if str(type(self)) == "<class 'pymrio.core.mriosystem.IOSystem'>":
            _tmp.meta.note('IOSystem copy {new} based on {old}'.format(
                new=new_name, old=self.meta.name))
            _tmp.meta.change_meta('name', new_name, log=False)
        else:
            _tmp.name = new_name
        return _tmp


Instruction:
Create a Python 3 function for
returning names of y cat. of the IOSystem as unique names in order is available.

Response:
def get_Y_categories(self, entries=None):
        """ Returns names of y cat. of the IOSystem as unique names in order

        Parameters
        ----------
        entries : List, optional
            If given, retuns an list with None for all values not in entries.

        Returns
        -------
        Index
            List of categories, None if no attribute to determine
            list is available
        """
        possible_dataframes = ['Y', 'FY']

        for df in possible_dataframes:
            if (df in self.__dict__) and (getattr(self, df) is not None):
                try:
                    ind = getattr(self, df).columns.get_level_values(
                        'category').unique()
                except (AssertionError, KeyError):
                    ind = getattr(self, df).columns.get_level_values(
                        1).unique()
                if entries:
                    if type(entries) is str:
                        entries = [entries]
                    ind = ind.tolist()
                    return [None if ee not in entries else ee for ee in ind]
                else:
                    return ind
        else:
            logging.warn("No attributes available to get Y categories")
            return None


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_index(self, as_dict=False, grouping_pattern=None):

        possible_dataframes = ['A', 'L', 'Z', 'Y', 'F', 'FY', 'M', 'S',
                               'D_cba', 'D_pba',  'D_imp', 'D_exp',
                               'D_cba_reg', 'D_pba_reg',
                               'D_imp_reg', 'D_exp_reg',
                               'D_cba_cap', 'D_pba_cap',
                               'D_imp_cap', 'D_exp_cap', ]
        for df in possible_dataframes:
            if (df in self.__dict__) and (getattr(self, df) is not None):
                orig_idx = getattr(self, df).index
                break
        else:
            logging.warn("No attributes available to get index")
            return None

        if as_dict:
            dd = {k: k for k in orig_idx}
            if grouping_pattern:
                for pattern, new_group in grouping_pattern.items():
                    if type(pattern) is str:
                        dd.update({k: new_group for k, v in dd.items() if
                                   re.match(pattern, k)})
                    else:
                        dd.update({k: new_group for k, v in dd.items() if
                                   all([re.match(pat, k[nr])
                                        for nr, pat in enumerate(pattern)])})
            return dd

        else:
            return orig_idx

Response: Returns the index of the dataframes in the system.


Instruction:
How would you implement a function in Python 3 that
sets the pd dataframe index of all dataframes in the system to index

Response:
def set_index(self, index):
        """ Sets the pd dataframe index of all dataframes in the system to index
        """
        for df in self.get_DataFrame(data=True, with_population=False):
            df.index = index


Instruction:
How would you code a function in Python 3 to
return a generator which yields all panda. DataFrames or there names of the DataFrames.

Response:
def get_DataFrame(self, data=False, with_unit=True, with_population=True):
        """ Yields all panda.DataFrames or there names

        Notes
        -----
        For IOSystem this does not include the DataFrames in the extensions.

        Parameters
        ----------
        data : boolean, optional
            If True, returns a generator which yields the DataFrames.
            If False, returns a generator which
            yields only the names of the DataFrames

        with_unit: boolean, optional
            If True, includes the 'unit' DataFrame
            If False, does not include the 'unit' DataFrame.
            The method than only yields the numerical data tables

        with_population: boolean, optional
            If True, includes the 'population' vector
            If False, does not include the 'population' vector.

        Returns
        -------
            DataFrames or string generator, depending on parameter data

        """

        for key in self.__dict__:
            if (key is 'unit') and not with_unit:
                continue
            if (key is 'population') and not with_population:
                continue
            if type(self.__dict__[key]) is pd.DataFrame:
                if data:
                    yield getattr(self, key)
                else:
                    yield key


Instruction:
Write a Python 3 script to
save the system to path.

Response:
def save(self, path, table_format='txt', sep='\t',
             table_ext=None, float_format='%.12g'):
        """ Saving the system to path

        Parameters
        ----------
        path : pathlib.Path or string
            path for the saved data (will be created if necessary, data
            within will be overwritten).

        table_format : string
            Format to save the DataFrames:

                - 'pkl' : Binary pickle files,
                          alias: 'pickle', 'bin', 'binary'
                - 'txt' : Text files (default), alias: 'text', 'csv'

        table_ext : string, optional
            File extension,
            default depends on table_format(.pkl for pickle, .txt for text)

        sep : string, optional
            Field delimiter for the output file, only for txt files.
            Default: tab ('\t')

        float_format : string, optional
            Format for saving the DataFrames,
            default = '%.12g', only for txt files
        """

        if type(path) is str:
            path = path.rstrip('\\')
            path = Path(path)

        path.mkdir(parents=True, exist_ok=True)

        para_file_path = path / DEFAULT_FILE_NAMES['filepara']
        file_para = dict()
        file_para['files'] = dict()

        if table_format in ['text', 'csv', 'txt']:
            table_format = 'txt'
        elif table_format in ['pickle', 'bin', 'binary', 'pkl']:
            table_format = 'pkl'
        else:
            raise ValueError('Unknown table format "{}" - '
                             'must be "txt" or "pkl"'.format(table_format))
            return None

        if not table_ext:
            if table_format == 'txt':
                table_ext = '.txt'
            if table_format == 'pkl':
                table_ext = '.pkl'

        if str(type(self)) == "<class 'pymrio.core.mriosystem.IOSystem'>":
            file_para['systemtype'] = GENERIC_NAMES['iosys']
        elif str(type(self)) == "<class 'pymrio.core.mriosystem.Extension'>":
            file_para['systemtype'] = GENERIC_NAMES['ext']
            file_para['name'] = self.name
        else:
            logging.warn('Unknown system type {} - set to "undef"'.format(
                str(type(self))))
            file_para['systemtype'] = 'undef'

        for df, df_name in zip(self.get_DataFrame(data=True),
                               self.get_DataFrame()):
            if type(df.index) is pd.MultiIndex:
                nr_index_col = len(df.index.levels)
            else:
                nr_index_col = 1

            if type(df.columns) is pd.MultiIndex:
                nr_header = len(df.columns.levels)
            else:
                nr_header = 1

            save_file = df_name + table_ext
            save_file_with_path = path / save_file
            logging.info('Save file {}'.format(save_file_with_path))
            if table_format == 'txt':
                df.to_csv(save_file_with_path, sep=sep,
                          float_format=float_format)
            else:
                df.to_pickle(save_file_with_path)

            file_para['files'][df_name] = dict()
            file_para['files'][df_name]['name'] = save_file
            file_para['files'][df_name]['nr_index_col'] = str(nr_index_col)
            file_para['files'][df_name]['nr_header'] = str(nr_header)

        with para_file_path.open(mode='w') as pf:
            json.dump(file_para, pf, indent=4)

        if file_para['systemtype'] == GENERIC_NAMES['iosys']:
            if not self.meta:
                self.meta = MRIOMetaData(name=self.name,
                                         location=path)

            self.meta._add_fileio("Saved {} to {}".format(self.name, path))
            self.meta.save(location=path)

        return self


Instruction:
Can you write a function in Python 3 where it
sets new names for the regions in the current object

Response:
def rename_regions(self, regions):
        """ Sets new names for the regions

        Parameters
        ----------
        regions : list or dict
            In case of dict: {'old_name' :  'new_name'} with a
                entry for each old_name which should be renamed
            In case of list: List of new names in order and complete
                without repetition

        """

        if type(regions) is list:
            regions = {old: new for old, new in
                       zip(self.get_regions(), regions)}

        for df in self.get_DataFrame(data=True):
            df.rename(index=regions, columns=regions, inplace=True)

        try:
            for ext in self.get_extensions(data=True):
                for df in ext.get_DataFrame(data=True):
                    df.rename(index=regions, columns=regions, inplace=True)
        except:
            pass

        self.meta._add_modify("Changed country names")
        return self


Instruction:
Can you generate a brief explanation for the following Python 3 code
def rename_sectors(self, sectors):

        if type(sectors) is list:
            sectors = {old: new for old, new in
                       zip(self.get_sectors(), sectors)}

        for df in self.get_DataFrame(data=True):
            df.rename(index=sectors, columns=sectors, inplace=True)

        try:
            for ext in self.get_extensions(data=True):
                for df in ext.get_DataFrame(data=True):
                    df.rename(index=sectors, columns=sectors, inplace=True)
        except:
            pass
        self.meta._add_modify("Changed sector names")
        return self

Response: Sets new names for the sectors in the current instance


Instruction:
Explain what the following Python 3 code does
def rename_Y_categories(self, Y_categories):

        if type(Y_categories) is list:
            Y_categories = {old: new for old, new in
                            zip(self.get_Y_categories(), Y_categories)}

        for df in self.get_DataFrame(data=True):
            df.rename(index=Y_categories, columns=Y_categories, inplace=True)

        try:
            for ext in self.get_extensions(data=True):
                for df in ext.get_DataFrame(data=True):
                    df.rename(index=Y_categories,
                              columns=Y_categories,
                              inplace=True)
        except:
            pass

        self.meta._add_modify("Changed Y category names")
        return self

Response: Sets new names for the Y_categories column in the DataFrame and all extensions.