Instruction:
Write a Python 3 script to
retry calling the decorated function using an exponential backoff. :param callback_by_exception: callback/method invocation on certain exceptions :type callback_by_exception: None or dict

Response:
def retry(ExceptionToCheck, tries=10, timeout_secs=1.0, logger=None, callback_by_exception=None):
    """
    Retry calling the decorated function using an exponential backoff.
    :param callback_by_exception: callback/method invocation on certain exceptions
    :type callback_by_exception: None or dict
    """
    def deco_retry(f):
        def f_retry(*args, **kwargs):
            mtries, mdelay = tries, timeout_secs
            run_one_last_time = True
            while mtries > 1:
                try:
                    return f(*args, **kwargs)
                except ExceptionToCheck as e:
                    # check if this exception is something the caller wants special handling for
                    callback_errors = callback_by_exception or {}
                    for error_type in callback_errors:
                        if isinstance(e, error_type):
                            callback_logic = callback_by_exception[error_type]
                            should_break_out = run_one_last_time = False
                            if isinstance(callback_logic, (list, tuple)):
                                callback_logic, should_break_out = callback_logic
                                if isinstance(should_break_out, (list, tuple)):
                                    should_break_out, run_one_last_time = should_break_out
                            callback_logic()
                            if should_break_out:  # caller requests we stop handling this exception
                                break
                    # traceback.print_exc()
                    half_interval = mdelay * 0.10  # interval size
                    actual_delay = random.uniform(mdelay - half_interval, mdelay + half_interval)
                    msg = "Retrying in %.2f seconds ..." % actual_delay
                    if logger is None:
                        logging.exception(msg)
                    else:
                        logger.exception(msg)
                    time.sleep(actual_delay)
                    mtries -= 1
                    mdelay *= 2
            if run_one_last_time:  # one exception may be all the caller wanted in certain cases
                return f(*args, **kwargs)
        return f_retry  # true decorator
    return deco_retry


Instruction:
Implement a function in Python 3 to
build RTSP url for stream.

Response:
def stream_url(self):
        """Build url for stream."""
        rtsp_url = RTSP_URL.format(
            host=self.config.host, video=self.video_query,
            audio=self.audio_query, event=self.event_query)
        _LOGGER.debug(rtsp_url)
        return rtsp_url


Instruction:
How would you explain what the following Python 3 function does
def session_callback(self, signal):
        if signal == SIGNAL_DATA:
            self.event.new_event(self.data)

        elif signal == SIGNAL_FAILED:
            self.retry()

        if signal in [SIGNAL_PLAYING, SIGNAL_FAILED] and \
                self.connection_status_callback:
            self.connection_status_callback(signal)

Response: Signal from stream session.


Instruction:
Here you have a function in Python 3, explain what it does
def retry(self):
        self.stream = None
        self.config.loop.call_later(RETRY_TIMER, self.start)
        _LOGGER.debug('Reconnecting to %s', self.config.host)

Response: Retry connection to the device.


Instruction:
How would you explain what the following Python 3 function does
def connect_mysql():
    MySQLConnection.get_characterset_info = MySQLConnection.get_charset

    db = create_engine(engine_name)
    db.echo = True
    db.connect()

    return db

Response: Connect to MySQL and return an inspector object


Instruction:
Explain what the following Python 3 code does
def check_perm(user_id, permission_code):
    try:
        perm = db.DBSession.query(Perm).filter(Perm.code==permission_code).one()
    except NoResultFound:
        raise PermissionError("Nonexistent permission type: %s"%(permission_code))

    try:
        res = db.DBSession.query(User).join(RoleUser, RoleUser.user_id==User.id).\
            join(Perm, Perm.id==perm.id).\
            join(RolePerm, RolePerm.perm_id==Perm.id).filter(User.id==user_id).one()
    except NoResultFound:
        raise PermissionError("Permission denied. User %s does not have permission %s"%
                        (user_id, permission_code))

Response: Checks whether a user has permission to perform an action.


Instruction:
Here you have a function in Python 3, explain what it does
def required_perms(*req_perms):
    def dec_wrapper(wfunc):
        @wraps(wfunc)
        def wrapped(*args, **kwargs):
            user_id = kwargs.get("user_id")
            for perm in req_perms:
                check_perm(user_id, perm)

            return wfunc(*args, **kwargs)

        return wrapped
    return dec_wrapper

Response: Decorator to check if user has required perms


Instruction:
How would you implement a function in Python 3 that
gives a time period name fetch the hydra - compatible time abbreviation.

Response:
def get_time_period(period_name):
    """
        Given a time period name, fetch the hydra-compatible time
        abbreviation.
    """
    time_abbreviation = time_map.get(period_name.lower())

    if time_abbreviation is None:
        raise Exception("Symbol %s not recognised as a time period"%period_name)

    return time_abbreviation


Instruction:
Can you implement a function in Python 3 that
converts a string timestamp into a date time.

Response:
def get_datetime(timestamp):
    """
        Turn a string timestamp into a date time. First tries to use dateutil.
        Failing that it tries to guess the time format and converts it manually
        using stfptime.

        @returns: A timezone unaware timestamp.
    """
    timestamp_is_float = False
    try:
        float(timestamp)
        timestamp_is_float = True
    except (ValueError, TypeError):
        pass

    if timestamp_is_float == True:
        raise ValueError("Timestamp %s is a float"%(timestamp,))

    #First try to use date util. Failing that, continue
    try:
        parsed_dt = parse(timestamp, dayfirst=False)
        if parsed_dt.tzinfo is None:
            return parsed_dt
        else:

            return parsed_dt.replace(tzinfo=None)
    except:
        pass

    if isinstance(timestamp, datetime):
        return timestamp

    fmt = guess_timefmt(timestamp)
    if fmt is None:
        fmt = FORMAT

    # and proceed as usual
    try:
        ts_time = datetime.strptime(timestamp, fmt)
    except ValueError as e:
        if e.message.split(' ', 1)[0].strip() == 'unconverted':
            utcoffset = e.message.split()[3].strip()
            timestamp = timestamp.replace(utcoffset, '')
            ts_time = datetime.strptime(timestamp, fmt)
            # Apply offset
            tzoffset = timedelta(hours=int(utcoffset[0:3]),
                                          minutes=int(utcoffset[3:5]))
            ts_time -= tzoffset
        else:
            raise e

    return ts_time


Instruction:
Write a Python 3 script to
convert a timestamp as defined in the soap interface to the time format stored in the database.

Response:
def timestamp_to_ordinal(timestamp):
    """Convert a timestamp as defined in the soap interface to the time format
    stored in the database.
    """

    if timestamp is None:
        return None

    ts_time = get_datetime(timestamp)
    # Convert time to Gregorian ordinal (1 = January 1st, year 1)
    ordinal_ts_time = Decimal(ts_time.toordinal())
    total_seconds = (ts_time -
                     datetime(ts_time.year,
                                       ts_time.month,
                                       ts_time.day,
                                       0, 0, 0)).total_seconds()

    fraction = (Decimal(repr(total_seconds)) / Decimal(86400)).quantize(Decimal('.00000000000000000001'),rounding=ROUND_HALF_UP)
    ordinal_ts_time += fraction
    log.debug("%s converted to %s", timestamp, ordinal_ts_time)

    return ordinal_ts_time


Instruction:
Can you generate a brief explanation for the following Python 3 code
def date_to_string(date, seasonal=False):

    seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999')
    if seasonal:
        FORMAT = seasonal_key+'-%m-%dT%H:%M:%S.%f'
    else:
        FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
    return date.strftime(FORMAT)

Response: Convert a date to a standard string used by Hydra.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def guess_timefmt(datestr):

    if isinstance(datestr, float) or isinstance(datestr, int):
        return None

    seasonal_key = str(config.get('DEFAULT', 'seasonal_key', '9999'))

    #replace 'T' with space to handle ISO times.
    if datestr.find('T') > 0:
        dt_delim = 'T'
    else:
        dt_delim = ' '

    delimiters = ['-', '.', ' ', '/']
    formatstrings = [['%Y', '%m', '%d'],
                     ['%d', '%m', '%Y'],
                     ['%d', '%b', '%Y'],
                     ['XXXX', '%m', '%d'],
                     ['%d', '%m', 'XXXX'],
                     ['%d', '%b', 'XXXX'],
                     [seasonal_key, '%m', '%d'],
                     ['%d', '%m', seasonal_key],
                     ['%d', '%b', seasonal_key]]

    timeformats = ['%H:%M:%S.%f', '%H:%M:%S', '%H:%M', '%H:%M:%S.%f000Z', '%H:%M:%S.%fZ']

    # Check if a time is indicated or not
    for timefmt in timeformats:
        try:
            datetime.strptime(datestr.split(dt_delim)[-1].strip(), timefmt)
            usetime = True
            break
        except ValueError:
            usetime = False

    # Check the simple ones:
    for fmt in formatstrings:
        for delim in delimiters:
            datefmt = fmt[0] + delim + fmt[1] + delim + fmt[2]
            if usetime:
                for timefmt in timeformats:
                    complfmt = datefmt + dt_delim + timefmt
                    try:
                        datetime.strptime(datestr, complfmt)
                        return complfmt
                    except ValueError:
                        pass
            else:
                try:
                    datetime.strptime(datestr, datefmt)
                    return datefmt
                except ValueError:
                    pass

    # Check for other formats:
    custom_formats = ['%d/%m/%Y', '%b %d %Y', '%B %d %Y','%d/%m/XXXX', '%d/%m/'+seasonal_key]

    for fmt in custom_formats:
        if usetime:
            for timefmt in timeformats:
                complfmt = fmt + dt_delim + timefmt
                try:
                    datetime.strptime(datestr, complfmt)
                    return complfmt
                except ValueError:
                    pass

        else:
            try:
                datetime.strptime(datestr, fmt)
                return fmt
            except ValueError:
                pass

    return None

Response: Try to guess the time format for a given date string.


Instruction:
How would you explain what the following Python 3 function does
def reindex_timeseries(ts_string, new_timestamps):
    #If a single timestamp is passed in, turn it into a list
    #Reindexing can't work if it's not a list
    if not isinstance(new_timestamps, list):
        new_timestamps = [new_timestamps]

    #Convert the incoming timestamps to datetimes
    #if they are not datetimes.
    new_timestamps_converted = []
    for t in new_timestamps:
        new_timestamps_converted.append(get_datetime(t))

    new_timestamps = new_timestamps_converted

    seasonal_year = config.get('DEFAULT','seasonal_year', '1678')
    seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999')

    ts = ts_string.replace(seasonal_key, seasonal_year)

    timeseries = pd.read_json(ts)

    idx = timeseries.index

    ts_timestamps = new_timestamps

    #'Fix' the incoming timestamp in case it's a seasonal value
    if type(idx) == pd.DatetimeIndex:
        if set(idx.year) == set([int(seasonal_year)]):
            if isinstance(new_timestamps,  list):
                seasonal_timestamp = []
                for t in ts_timestamps:
                    t_1900 = t.replace(year=int(seasonal_year))
                    seasonal_timestamp.append(t_1900)
                ts_timestamps = seasonal_timestamp

    #Reindex the timeseries to reflect the requested timestamps
    reindexed_ts = timeseries.reindex(ts_timestamps, method='ffill')

    i = reindexed_ts.index

    reindexed_ts.index = pd.Index(new_timestamps, names=i.names)

    #If there are no values at all, just return None
    if len(reindexed_ts.dropna()) == 0:
        return None

    #Replace all numpy NAN values with None
    pandas_ts = reindexed_ts.where(reindexed_ts.notnull(), None)

    return pandas_ts

Response: reindex a timeseries with the supplied timestamps


Instruction:
Here you have a function in Python 3, explain what it does
def parse_time_step(time_step, target='s', units_ref=None):
    log.info("Parsing time step %s", time_step)
    # export numerical value from string using regex
    value = re.findall(r'\d+', time_step)[0]
    valuelen = len(value)

    try:
        value = float(value)
    except:
        HydraPluginError("Unable to extract number of time steps (%s) from time step %s" % (value, time_step))

    unit = time_step[valuelen:].strip()

    period = get_time_period(unit)

    log.info("Time period is %s", period)

    converted_time_step = units_ref.convert(value, period, target)

    log.info("Time period is %s %s", converted_time_step, period)

    return float(converted_time_step), value, period

Response: Read in the time step and convert it to seconds.


Instruction:
Can you write a function in Python 3 where it
creates a list of datetimes based on an start time, end time and time step. If such a list is already passed in, then this is not necessary. Often either the start_time, end_time, time_step is passed into an app or the time_axis is passed in directly. This function returns a time_axis in both situations.

Response:
def get_time_axis(start_time, end_time, time_step, time_axis=None):
    """
        Create a list of datetimes based on an start time, end time and
        time step.  If such a list is already passed in, then this is not
        necessary.

        Often either the start_time, end_time, time_step is passed into an
        app or the time_axis is passed in directly. This function returns a
        time_axis in both situations.
    """

    #Do this import here to avoid a circular dependency
    from ..lib import units

    if time_axis is not None:
        actual_dates_axis = []
        for t in time_axis:
            #If the user has entered the time_axis with commas, remove them.
            t = t.replace(',', '').strip()
            if t == '':
                continue
            actual_dates_axis.append(get_datetime(t))
        return actual_dates_axis

    else:
        if start_time is None:
            raise HydraPluginError("A start time must be specified")
        if end_time is None:
            raise HydraPluginError("And end time must be specified")
        if time_step is None:
            raise HydraPluginError("A time-step must be specified")

        start_date = get_datetime(start_time)
        end_date = get_datetime(end_time)
        delta_t, value, output_units = parse_time_step(time_step, units_ref=units)

        time_axis = [start_date]

        value = int(value)
        while start_date < end_date:
            #Months and years are a special case, so treat them differently
            if(output_units.lower() == "mon"):
                start_date = start_date + relativedelta(months=value)
            elif (output_units.lower() == "yr"):
                start_date = start_date + relativedelta(years=value)
            else:
                start_date += timedelta(seconds=delta_t)
            time_axis.append(start_date)
        return time_axis


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_all_attributes(network):
    attrs = network.attributes
    for n in network.nodes:
        attrs.extend(n.attributes)
    for l in network.links:
        attrs.extend(l.attributes)
    for g in network.resourcegroups:
        attrs.extend(g.attributes)

    return attrs

Response: Get all the complex mode attributes in the network.


Instruction:
How would you implement a function in Python 3 that
adds a network to the database.

Response:
def add_network(network,**kwargs):
    """
    Takes an entire network complex model and saves it to the DB.  This
    complex model includes links & scenarios (with resource data).  Returns
    the network's complex model.

    As links connect two nodes using the node_ids, if the nodes are new
    they will not yet have node_ids. In this case, use negative ids as
    temporary IDS until the node has been given an permanent ID.

    All inter-object referencing of new objects should be done using
    negative IDs in the client.

    The returned object will have positive IDS

    """
    db.DBSession.autoflush = False

    start_time = datetime.datetime.now()
    log.debug("Adding network")

    insert_start = datetime.datetime.now()

    proj_i = db.DBSession.query(Project).filter(Project.id == network.project_id).first()
    if proj_i is None:
        raise HydraError("Project ID is none. A project ID must be specified on the Network")

    existing_net = db.DBSession.query(Network).filter(Network.project_id == network.project_id, Network.name==network.name).first()
    if existing_net is not None:
        raise HydraError("A network with the name %s is already in project %s"%(network.name, network.project_id))

    user_id = kwargs.get('user_id')
    proj_i.check_write_permission(user_id)

    net_i = Network()
    net_i.project_id          = network.project_id
    net_i.name                = network.name
    net_i.description         = network.description
    net_i.created_by          = user_id
    net_i.projection          = network.projection

    if network.layout is not None:
        net_i.layout = network.get_layout()

    network.id = net_i.id
    db.DBSession.add(net_i)
    db.DBSession.flush()
    #These two lists are used for comparison and lookup, so when
    #new attributes are added, these lists are extended.

    #List of all the resource attributes
    all_resource_attrs = {}

    name_map = {network.name:net_i}
    network_attrs, network_defaults = _bulk_add_resource_attrs(net_i.id, 'NETWORK', [network], name_map)
    hdb.add_resource_types(net_i, network.types)

    all_resource_attrs.update(network_attrs)

    log.info("Network attributes added in %s", get_timing(start_time))
    node_id_map, node_attrs, node_datasets = _add_nodes(net_i, network.nodes)
    all_resource_attrs.update(node_attrs)

    link_id_map, link_attrs, link_datasets = _add_links(net_i, network.links, node_id_map)
    all_resource_attrs.update(link_attrs)

    grp_id_map, grp_attrs, grp_datasets = _add_resource_groups(net_i, network.resourcegroups)
    all_resource_attrs.update(grp_attrs)

    defaults = list(grp_datasets.values()) + list(link_datasets.values()) \
        + list(node_datasets.values()) + list(network_defaults.values())

    start_time = datetime.datetime.now()

    scenario_names = []
    if network.scenarios is not None:
        log.info("Adding scenarios to network")
        for s in network.scenarios:

            if s.name in scenario_names:
                raise HydraError("Duplicate scenario name: %s"%(s.name))

            scen = Scenario()
            scen.name                 = s.name
            scen.description          = s.description
            scen.layout               = s.get_layout()
            scen.start_time           = str(timestamp_to_ordinal(s.start_time)) if s.start_time else None
            scen.end_time             = str(timestamp_to_ordinal(s.end_time)) if s.end_time else None
            scen.time_step            = s.time_step
            scen.created_by           = user_id

            scenario_names.append(s.name)

            #extract the data from each resourcescenario
            incoming_datasets = []
            scenario_resource_attrs = []
            for r_scen in s.resourcescenarios:
                ra = all_resource_attrs[r_scen.resource_attr_id]
                incoming_datasets.append(r_scen.dataset)
                scenario_resource_attrs.append(ra)

            data_start_time = datetime.datetime.now()

            for default in defaults:
                scen.add_resource_scenario(JSONObject(default),
                                           JSONObject({'id':default['dataset_id']}),
                                           source=kwargs.get('app_name'))

            datasets = data._bulk_insert_data(
                                              incoming_datasets,
                                              user_id,
                                              kwargs.get('app_name')
                                             )

            log.info("Data bulk insert took %s", get_timing(data_start_time))
            ra_start_time = datetime.datetime.now()
            for i, ra in enumerate(scenario_resource_attrs):
                scen.add_resource_scenario(ra, datasets[i], source=kwargs.get('app_name'))

            log.info("Resource scenarios added in  %s", get_timing(ra_start_time))

            item_start_time = datetime.datetime.now()
            if s.resourcegroupitems is not None:
                for group_item in s.resourcegroupitems:
                    group_item_i = ResourceGroupItem()
                    group_item_i.group = grp_id_map[group_item.group_id]
                    group_item_i.ref_key  = group_item.ref_key
                    if group_item.ref_key == 'NODE':
                        group_item_i.node = node_id_map[group_item.ref_id]
                    elif group_item.ref_key == 'LINK':
                        group_item_i.link = link_id_map[group_item.ref_id]
                    elif group_item.ref_key == 'GROUP':
                        group_item_i.subgroup = grp_id_map[group_item.ref_id]
                    else:
                        raise HydraError("A ref key of %s is not valid for a "
                                         "resource group item.",\
                                         group_item.ref_key)

                    scen.resourcegroupitems.append(group_item_i)
            log.info("Group items insert took %s", get_timing(item_start_time))
            net_i.scenarios.append(scen)

    log.info("Scenarios added in %s", get_timing(start_time))
    net_i.set_owner(user_id)

    db.DBSession.flush()
    log.info("Insertion of network took: %s",(datetime.datetime.now()-insert_start))

    return net_i


Instruction:
Can you write a function in Python 3 where it
gets all the resource attributes for a given network.

Response:
def _get_all_resource_attributes(network_id, template_id=None):
    """
        Get all the attributes for the nodes, links and groups of a network.
        Return these attributes as a dictionary, keyed on type (NODE, LINK, GROUP)
        then by ID of the node or link.
    """
    base_qry = db.DBSession.query(
                               ResourceAttr.id.label('id'),
                               ResourceAttr.ref_key.label('ref_key'),
                               ResourceAttr.cr_date.label('cr_date'),
                               ResourceAttr.attr_is_var.label('attr_is_var'),
                               ResourceAttr.node_id.label('node_id'),
                               ResourceAttr.link_id.label('link_id'),
                               ResourceAttr.group_id.label('group_id'),
                               ResourceAttr.network_id.label('network_id'),
                               ResourceAttr.attr_id.label('attr_id'),
                               Attr.name.label('name'),
                               Attr.dimension_id.label('dimension_id'),
                              ).filter(Attr.id==ResourceAttr.attr_id)

    all_node_attribute_qry = base_qry.join(Node).filter(Node.network_id==network_id)

    all_link_attribute_qry = base_qry.join(Link).filter(Link.network_id==network_id)

    all_group_attribute_qry = base_qry.join(ResourceGroup).filter(ResourceGroup.network_id==network_id)
    network_attribute_qry = base_qry.filter(ResourceAttr.network_id==network_id)

    #Filter the group attributes by template
    if template_id is not None:
        all_node_attribute_qry = all_node_attribute_qry.join(ResourceType).join(TemplateType).join(TypeAttr).filter(TemplateType.template_id==template_id).filter(ResourceAttr.attr_id==TypeAttr.attr_id)
        all_link_attribute_qry = all_link_attribute_qry.join(ResourceType).join(TemplateType).join(TypeAttr).filter(TemplateType.template_id==template_id).filter(ResourceAttr.attr_id==TypeAttr.attr_id)
        all_group_attribute_qry = all_group_attribute_qry.join(ResourceType).join(TemplateType).join(TypeAttr).filter(TemplateType.template_id==template_id).filter(ResourceAttr.attr_id==TypeAttr.attr_id)
        network_attribute_qry = network_attribute_qry.join(ResourceType, ResourceAttr.network_id==ResourceType.network_id).join(TemplateType).join(TypeAttr).filter(TemplateType.template_id==template_id).filter(ResourceAttr.attr_id==TypeAttr.attr_id)

    x = time.time()
    logging.info("Getting all attributes using execute")
    attribute_qry = all_node_attribute_qry.union(all_link_attribute_qry, all_group_attribute_qry, network_attribute_qry)
    all_attributes = db.DBSession.execute(attribute_qry.statement).fetchall()
    log.info("%s attrs retrieved in %s", len(all_attributes), time.time()-x)

    logging.info("Attributes retrieved. Processing results...")
    x = time.time()
    node_attr_dict = dict()
    link_attr_dict = dict()
    group_attr_dict = dict()
    network_attr_dict = dict()

    for attr in all_attributes:
        if attr.ref_key == 'NODE':
            nodeattr = node_attr_dict.get(attr.node_id, [])
            nodeattr.append(attr)
            node_attr_dict[attr.node_id] = nodeattr
        elif attr.ref_key == 'LINK':
            linkattr = link_attr_dict.get(attr.link_id, [])
            linkattr.append(attr)
            link_attr_dict[attr.link_id] = linkattr
        elif attr.ref_key == 'GROUP':
            groupattr = group_attr_dict.get(attr.group_id, [])
            groupattr.append(attr)
            group_attr_dict[attr.group_id] = groupattr
        elif attr.ref_key == 'NETWORK':
            networkattr = network_attr_dict.get(attr.network_id, [])
            networkattr.append(attr)
            network_attr_dict[attr.network_id] = networkattr

    all_attributes = {
        'NODE' : node_attr_dict,
        'LINK' : link_attr_dict,
        'GROUP': group_attr_dict,
        'NETWORK': network_attr_dict,
    }

    logging.info("Attributes processed in %s", time.time()-x)
    return all_attributes


Instruction:
Here you have a function in Python 3, explain what it does
def _get_all_templates(network_id, template_id):
    base_qry = db.DBSession.query(
                               ResourceType.ref_key.label('ref_key'),
                               ResourceType.node_id.label('node_id'),
                               ResourceType.link_id.label('link_id'),
                               ResourceType.group_id.label('group_id'),
                               ResourceType.network_id.label('network_id'),
                               Template.name.label('template_name'),
                               Template.id.label('template_id'),
                               TemplateType.id.label('type_id'),
                               TemplateType.layout.label('layout'),
                               TemplateType.name.label('type_name'),
                              ).filter(TemplateType.id==ResourceType.type_id,
                                       Template.id==TemplateType.template_id)

    all_node_type_qry = base_qry.filter(Node.id==ResourceType.node_id,
                                        Node.network_id==network_id)

    all_link_type_qry = base_qry.filter(Link.id==ResourceType.link_id,
                                        Link.network_id==network_id)

    all_group_type_qry = base_qry.filter(ResourceGroup.id==ResourceType.group_id,
                                         ResourceGroup.network_id==network_id)

    network_type_qry = base_qry.filter(ResourceType.network_id==network_id)

    #Filter the group attributes by template
    if template_id is not None:
        all_node_type_qry = all_node_type_qry.filter(Template.id==template_id)
        all_link_type_qry = all_link_type_qry.filter(Template.id==template_id)
        all_group_type_qry = all_group_type_qry.filter(Template.id==template_id)

    x = time.time()
    log.info("Getting all types")
    type_qry = all_node_type_qry.union(all_link_type_qry, all_group_type_qry, network_type_qry)
    all_types = db.DBSession.execute(type_qry.statement).fetchall()
    log.info("%s types retrieved in %s", len(all_types), time.time()-x)

    log.info("Attributes retrieved. Processing results...")
    x = time.time()
    node_type_dict = dict()
    link_type_dict = dict()
    group_type_dict = dict()
    network_type_dict = dict()

    for t in all_types:
        templatetype = JSONObject({
                                    'template_id':t.template_id,
                                   'id':t.type_id,
                                   'template_name':t.template_name,
                                   'layout': t.layout,
                                   'name': t.type_name,})

        if t.ref_key == 'NODE':
            nodetype = node_type_dict.get(t.node_id, [])
            nodetype.append(templatetype)
            node_type_dict[t.node_id] = nodetype
        elif t.ref_key == 'LINK':
            linktype = link_type_dict.get(t.link_id, [])
            linktype.append(templatetype)
            link_type_dict[t.link_id] = linktype
        elif t.ref_key == 'GROUP':
            grouptype = group_type_dict.get(t.group_id, [])
            grouptype.append(templatetype)
            group_type_dict[t.group_id] = grouptype
        elif t.ref_key == 'NETWORK':
            nettype = network_type_dict.get(t.network_id, [])
            nettype.append(templatetype)
            network_type_dict[t.network_id] = nettype

    all_types = {
        'NODE' : node_type_dict,
        'LINK' : link_type_dict,
        'GROUP': group_type_dict,
        'NETWORK': network_type_dict,
    }

    logging.info("Attributes processed in %s", time.time()-x)
    return all_types

Response: Get all the templates for a given network and template.


Instruction:
Can you tell what is the following Python 3 function doing
def _get_all_group_items(network_id):
    base_qry = db.DBSession.query(ResourceGroupItem)

    item_qry = base_qry.join(Scenario).filter(Scenario.network_id==network_id)

    x = time.time()
    logging.info("Getting all items")
    all_items = db.DBSession.execute(item_qry.statement).fetchall()
    log.info("%s groups jointly retrieved in %s", len(all_items), time.time()-x)

    logging.info("items retrieved. Processing results...")
    x = time.time()
    item_dict = dict()
    for item in all_items:

        items = item_dict.get(item.scenario_id, [])
        items.append(item)
        item_dict[item.scenario_id] = items

    logging.info("items processed in %s", time.time()-x)

    return item_dict

Response: Get all the resource group items in the network across all scenarios


Instruction:
How would you code a function in Python 3 to
get all the resource scenarios in a network across all scenarios

Response:
def _get_all_resourcescenarios(network_id, user_id):
    """
        Get all the resource scenarios in a network, across all scenarios
        returns a dictionary of dict objects, keyed on scenario_id
    """

    rs_qry = db.DBSession.query(
                Dataset.type,
                Dataset.unit_id,
                Dataset.name,
                Dataset.hash,
                Dataset.cr_date,
                Dataset.created_by,
                Dataset.hidden,
                Dataset.value,
                ResourceScenario.dataset_id,
                ResourceScenario.scenario_id,
                ResourceScenario.resource_attr_id,
                ResourceScenario.source,
                ResourceAttr.attr_id,
    ).outerjoin(DatasetOwner, and_(DatasetOwner.dataset_id==Dataset.id, DatasetOwner.user_id==user_id)).filter(
                or_(Dataset.hidden=='N', Dataset.created_by==user_id, DatasetOwner.user_id != None),
                ResourceAttr.id == ResourceScenario.resource_attr_id,
                Scenario.id==ResourceScenario.scenario_id,
                Scenario.network_id==network_id,
                Dataset.id==ResourceScenario.dataset_id)

    x = time.time()
    logging.info("Getting all resource scenarios")
    all_rs = db.DBSession.execute(rs_qry.statement).fetchall()
    log.info("%s resource scenarios retrieved in %s", len(all_rs), time.time()-x)

    logging.info("resource scenarios retrieved. Processing results...")
    x = time.time()
    rs_dict = dict()
    for rs in all_rs:
        rs_obj = JSONObject(rs)
        rs_attr = JSONObject({'attr_id':rs.attr_id})

        value = rs.value

        rs_dataset = JSONDataset({
            'id':rs.dataset_id,
            'type' : rs.type,
            'unit_id' : rs.unit_id,
            'name' : rs.name,
            'hash' : rs.hash,
            'cr_date':rs.cr_date,
            'created_by':rs.created_by,
            'hidden':rs.hidden,
            'value':value,
            'metadata':{},
        })
        rs_obj.resourceattr = rs_attr
        rs_obj.value = rs_dataset
        rs_obj.dataset = rs_dataset

        scenario_rs = rs_dict.get(rs.scenario_id, [])
        scenario_rs.append(rs_obj)
        rs_dict[rs.scenario_id] = scenario_rs

    logging.info("resource scenarios processed in %s", time.time()-x)

    return rs_dict


Instruction:
Can you tell what is the following Python 3 function doing
def _get_metadata(network_id, user_id):
    log.info("Getting Metadata")
    dataset_qry = db.DBSession.query(
                Dataset
    ).outerjoin(DatasetOwner, and_(DatasetOwner.dataset_id==Dataset.id, DatasetOwner.user_id==user_id)).filter(
                or_(Dataset.hidden=='N', DatasetOwner.user_id != None),
                Scenario.id==ResourceScenario.scenario_id,
                Scenario.network_id==network_id,
                Dataset.id==ResourceScenario.dataset_id).distinct().subquery()

    rs_qry = db.DBSession.query(
                Metadata
    ).join(dataset_qry, Metadata.dataset_id==dataset_qry.c.id)

    x = time.time()
    logging.info("Getting all matadata")
    all_metadata = db.DBSession.execute(rs_qry.statement).fetchall()
    log.info("%s metadata jointly retrieved in %s",len(all_metadata), time.time()-x)

    logging.info("metadata retrieved. Processing results...")
    x = time.time()
    metadata_dict = dict()
    for m in all_metadata:
        if metadata_dict.get(m.dataset_id):
            metadata_dict[m.dataset_id][m.key] = six.text_type(m.value)
        else:
            metadata_dict[m.dataset_id] = {m.key : six.text_type(m.value)}

    logging.info("metadata processed in %s", time.time()-x)

    return metadata_dict

Response: Get all the metadata in a network across all scenarios


Instruction:
Make a summary of the following Python 3 code
def _get_network_owners(network_id):
    owners_i = db.DBSession.query(NetworkOwner).filter(
                        NetworkOwner.network_id==network_id).options(noload('network')).options(joinedload_all('user')).all()

    owners = [JSONObject(owner_i) for owner_i in owners_i]

    return owners

Response: Get all the nodes in a network


Instruction:
Here you have a function in Python 3, explain what it does
def _get_nodes(network_id, template_id=None):
    extras = {'types':[], 'attributes':[]}

    node_qry = db.DBSession.query(Node).filter(
                        Node.network_id==network_id,
                        Node.status=='A').options(
                            noload('network')
                        )
    if template_id is not None:
        node_qry = node_qry.filter(ResourceType.node_id==Node.id,
                                   TemplateType.id==ResourceType.type_id,
                                   TemplateType.template_id==template_id)
    node_res = db.DBSession.execute(node_qry.statement).fetchall()

    nodes = []
    for n in node_res:
        nodes.append(JSONObject(n, extras=extras))

    return nodes

Response: Get all the nodes in a network


Instruction:
How would you implement a function in Python 3 that
gets all the links in a network

Response:
def _get_links(network_id, template_id=None):
    """
        Get all the links in a network
    """
    extras = {'types':[], 'attributes':[]}
    link_qry = db.DBSession.query(Link).filter(
                                        Link.network_id==network_id,
                                        Link.status=='A').options(
                                            noload('network')
                                        )
    if template_id is not None:
        link_qry = link_qry.filter(ResourceType.link_id==Link.id,
                                   TemplateType.id==ResourceType.type_id,
                                   TemplateType.template_id==template_id)

    link_res = db.DBSession.execute(link_qry.statement).fetchall()

    links = []
    for l in link_res:
        links.append(JSONObject(l, extras=extras))

    return links


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_groups(network_id, template_id=None):
    extras = {'types':[], 'attributes':[]}
    group_qry = db.DBSession.query(ResourceGroup).filter(
                                        ResourceGroup.network_id==network_id,
                                        ResourceGroup.status=='A').options(
                                            noload('network')
                                        )

    if template_id is not None:
        group_qry = group_qry.filter(ResourceType.group_id==ResourceGroup.id,
                                     TemplateType.id==ResourceType.type_id,
                                     TemplateType.template_id==template_id)

    group_res = db.DBSession.execute(group_qry.statement).fetchall()
    groups = []
    for g in group_res:
        groups.append(JSONObject(g, extras=extras))

    return groups

Response: Get all the resource groups in a network


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_scenarios(network_id, include_data, user_id, scenario_ids=None):
    scen_qry = db.DBSession.query(Scenario).filter(
                    Scenario.network_id == network_id).options(
                        noload('network')).filter(
                        Scenario.status == 'A')

    if scenario_ids:
        logging.info("Filtering by scenario_ids %s",scenario_ids)
        scen_qry = scen_qry.filter(Scenario.id.in_(scenario_ids))
    extras = {'resourcescenarios': [], 'resourcegroupitems': []}
    scens = [JSONObject(s,extras=extras) for s in db.DBSession.execute(scen_qry.statement).fetchall()]

    all_resource_group_items = _get_all_group_items(network_id)

    if include_data == 'Y' or include_data == True:
        all_rs = _get_all_resourcescenarios(network_id, user_id)
        metadata = _get_metadata(network_id, user_id)

    for s in scens:
        s.resourcegroupitems = all_resource_group_items.get(s.id, [])

        if include_data == 'Y' or include_data == True:
            s.resourcescenarios  = all_rs.get(s.id, [])

            for rs in s.resourcescenarios:
                rs.dataset.metadata = metadata.get(rs.dataset_id, {})

    return scens

Response: Get all the scenarios in a network


Instruction:
Implement a Python 3 function for
getting a single network as a dictionary.

Response:
def get_network(network_id, summary=False, include_data='N', scenario_ids=None, template_id=None, **kwargs):
    """
        Return a whole network as a dictionary.
        network_id: ID of the network to retrieve
        include_data: 'Y' or 'N'. Indicate whether scenario data is to be returned.
                      This has a significant speed impact as retrieving large amounts
                      of data can be expensive.
        scenario_ids: list of IDS to be returned. Used if a network has multiple
                      scenarios but you only want one returned. Using this filter
                      will speed up this function call.
        template_id:  Return the network with only attributes associated with this
                      template on the network, groups, nodes and links.
    """
    log.debug("getting network %s"%network_id)
    user_id = kwargs.get('user_id')

    network_id = int(network_id)

    try:
        log.debug("Querying Network %s", network_id)
        net_i = db.DBSession.query(Network).filter(
                                Network.id == network_id).options(
                                noload('scenarios')).options(
                                noload('nodes')).options(
                                noload('links')).options(
                                noload('types')).options(
                                noload('attributes')).options(
                                noload('resourcegroups')).one()

        net_i.check_read_permission(user_id)

        net = JSONObject(net_i)

        net.nodes          = _get_nodes(network_id, template_id=template_id)
        net.links          = _get_links(network_id, template_id=template_id)
        net.resourcegroups = _get_groups(network_id, template_id=template_id)
        net.owners         = _get_network_owners(network_id)

        if summary is False:
            all_attributes = _get_all_resource_attributes(network_id, template_id)
            log.info("Setting attributes")
            net.attributes = all_attributes['NETWORK'].get(network_id, [])
            for node_i in net.nodes:
                node_i.attributes = all_attributes['NODE'].get(node_i.id, [])
            log.info("Node attributes set")
            for link_i in net.links:
                link_i.attributes = all_attributes['LINK'].get(link_i.id, [])
            log.info("Link attributes set")
            for group_i in net.resourcegroups:
                group_i.attributes = all_attributes['GROUP'].get(group_i.id, [])
            log.info("Group attributes set")

        log.info("Setting types")
        all_types = _get_all_templates(network_id, template_id)
        net.types = all_types['NETWORK'].get(network_id, [])
        for node_i in net.nodes:
            node_i.types = all_types['NODE'].get(node_i.id, [])
        for link_i in net.links:
            link_i.types = all_types['LINK'].get(link_i.id, [])
        for group_i in net.resourcegroups:
            group_i.types = all_types['GROUP'].get(group_i.id, [])

        log.info("Getting scenarios")

        net.scenarios = _get_scenarios(network_id, include_data, user_id, scenario_ids)

    except NoResultFound:
        raise ResourceNotFoundError("Network (network_id=%s) not found." %
                                  network_id)

    return net


Instruction:
Create a Python 3 function for
getting all the nodes in a network.

Response:
def get_nodes(network_id, template_id=None, **kwargs):
    """
        Get all the nodes in a network.
        args:
            network_id (int): The network in which to search
            template_id (int): Only return nodes whose type is in this template.
    """
    user_id = kwargs.get('user_id')
    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
        net_i.check_read_permission(user_id=user_id)
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))

    node_qry = db.DBSession.query(Node).filter(
                        Node.network_id==network_id,
                        Node.status=='A').options(
                            noload('network')
                        ).options(
                            joinedload_all('types.templatetype')
                        ).options(
                            joinedload_all('attributes.attr')
                        )
    if template_id is not None:
        node_qry = node_qry.filter(ResourceType.node_id==Node.id,
                                   TemplateType.id==ResourceType.type_id,
                                   TemplateType.template_id==template_id)
    nodes = node_qry.all()

    return nodes


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_links(network_id, template_id=None, **kwargs):
    user_id = kwargs.get('user_id')
    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
        net_i.check_read_permission(user_id=user_id)
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))

    link_qry = db.DBSession.query(Link).filter(
                                        Link.network_id==network_id,
                                        Link.status=='A').options(
                                            noload('network')
                                        ).options(
                                            joinedload_all('types.templatetype')
                                        ).options(
                                            joinedload_all('attributes.attr')
                                        )

    if template_id is not None:
        link_qry = link_qry.filter(ResourceType.link_id==Link.id,
                                   TemplateType.id==ResourceType.type_id,
                                   TemplateType.template_id==template_id)

    links = link_qry.all()
    return links

Response: Get all the links in a network.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_groups(network_id, template_id=None, **kwargs):
    user_id = kwargs.get('user_id')
    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
        net_i.check_read_permission(user_id=user_id)
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))

    group_qry = db.DBSession.query(ResourceGroup).filter(
                                        ResourceGroup.network_id==network_id,
                                        ResourceGroup.status=='A').options(
                                            noload('network')
                                        ).options(
                                            joinedload_all('types.templatetype')
                                        ).options(
                                            joinedload_all('attributes.attr')
                                        )
    if template_id is not None:
        group_qry = group_qry.filter(ResourceType.group_id==ResourceGroup.id,
                                     TemplateType.id==ResourceType.type_id,
                                     TemplateType.template_id==template_id)

    groups = group_qry.all()

    return groups

Response: Get all the resource groups in a network.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_network_by_name(project_id, network_name,**kwargs):

    try:
        res = db.DBSession.query(Network.id).filter(func.lower(Network.name).like(network_name.lower()), Network.project_id == project_id).one()
        net = get_network(res.id, 'Y', None, **kwargs)
        return net
    except NoResultFound:
        raise ResourceNotFoundError("Network with name %s not found"%(network_name))

Response: Get a network as a complex model.


Instruction:
How would you implement a function in Python 3 that
checks if a network exists in the database.

Response:
def network_exists(project_id, network_name,**kwargs):
    """
    Return a whole network as a complex model.
    """
    try:
        db.DBSession.query(Network.id).filter(func.lower(Network.name).like(network_name.lower()), Network.project_id == project_id).one()
        return 'Y'
    except NoResultFound:
        return 'N'


Instruction:
Can you implement a function in Python 3 that
updates a network in the database.

Response:
def update_network(network,
    update_nodes = True,
    update_links = True,
    update_groups = True,
    update_scenarios = True,
    **kwargs):
    """
        Update an entire network
    """
    log.info("Updating Network %s", network.name)
    user_id = kwargs.get('user_id')
    #check_perm('update_network')

    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network.id).one()
    except NoResultFound:
        raise ResourceNotFoundError("Network with id %s not found"%(network.id))

    net_i.project_id          = network.project_id
    net_i.name                = network.name
    net_i.description         = network.description
    net_i.projection          = network.projection
    net_i.layout              = network.get_layout()

    all_resource_attrs = {}
    new_network_attributes = _update_attributes(net_i, network.attributes)
    all_resource_attrs.update(new_network_attributes)
    hdb.add_resource_types(net_i, network.types)

    #Maps temporary node_ids to real node_ids
    node_id_map = dict()

    if network.nodes is not None and update_nodes is True:
        log.info("Updating nodes")
        t0 = time.time()
        #First add all the nodes
        node_id_map = dict([(n.id, n) for n in net_i.nodes])
        for node in network.nodes:
            #If we get a negative or null node id, we know
            #it is a new node.
            if node.id is not None and node.id > 0:
                n = node_id_map[node.id]
                n.name        = node.name
                n.description = node.description
                n.x           = node.x
                n.y           = node.y
                n.status      = node.status
                n.layout      = node.get_layout()
            else:
                log.info("Adding new node %s", node.name)
                n = net_i.add_node(node.name,
                                   node.description,
                                   node.get_layout(),
                                   node.x,
                                   node.y)
                net_i.nodes.append(n)
                node_id_map[n.id] = n

            all_resource_attrs.update(_update_attributes(n, node.attributes))
            hdb.add_resource_types(n, node.types)
        log.info("Updating nodes took %s", time.time() - t0)

    link_id_map = dict()
    if network.links is not None and update_links is True:
        log.info("Updating links")
        t0 = time.time()
        link_id_map = dict([(l.link_id, l) for l in net_i.links])
        for link in network.links:
            node_1 = node_id_map[link.node_1_id]

            node_2 = node_id_map[link.node_2_id]

            if link.id is None or link.id < 0:
                log.info("Adding new link %s", link.name)
                l = net_i.add_link(link.name,
                                   link.description,
                                   link.get_layout(),
                                   node_1,
                                   node_2)
                net_i.links.append(l)
                link_id_map[link.id] = l
            else:
                l = link_id_map[link.id]
                l.name       = link.name
                l.link_descripion = link.description
                l.node_a          = node_1
                l.node_b          = node_2
                l.layout          = link.get_layout()

            all_resource_attrs.update(_update_attributes(l, link.attributes))
            hdb.add_resource_types(l, link.types)
        log.info("Updating links took %s", time.time() - t0)

    group_id_map = dict()
    #Next all the groups
    if network.resourcegroups is not None and update_groups is True:
        log.info("Updating groups")
        t0 = time.time()
        group_id_map = dict([(g.group_id, g) for g in net_i.resourcegroups])
        for group in network.resourcegroups:
            #If we get a negative or null group id, we know
            #it is a new group.
            if group.id is not None and group.id > 0:
                g_i = group_id_map[group.id]
                g_i.name        = group.name
                g_i.description = group.description
                g_i.status           = group.status
            else:
                log.info("Adding new group %s", group.name)
                g_i = net_i.add_group(group.name,
                                   group.description,
                                   group.status)
                net_i.resourcegroups.append(net_i)
                group_id_map[g_i.group_id] = g_i

            all_resource_attrs.update(_update_attributes(g_i, group.attributes))
            hdb.add_resource_types(g_i, group.types)
            group_id_map[group.id] = g_i
        log.info("Updating groups took %s", time.time() - t0)

    errors = []
    if network.scenarios is not None and update_scenarios is True:
        for s in network.scenarios:
            add_scenario = False
            if s.id is not None:
                if s.id > 0:
                    try:
                        scen_i = db.DBSession.query(Scenario).filter(Scenario.id==s.id).one()
                        if scen_i.locked == 'Y':
                            errors.append('Scenario %s was not updated as it is locked'%(s.id))
                            continue

                        scenario.update_scenario(s, flush=False, **kwargs)
                    except NoResultFound:
                        raise ResourceNotFoundError("Scenario %s not found"%(s.id))
                else:
                    add_scenario = True
            else:
                add_scenario = True

            if add_scenario is True:
                log.info("Adding new scenario %s to network", s.name)
                scenario.add_scenario(network.id, s, **kwargs)

    db.DBSession.flush()

    updated_net = get_network(network.id, summary=True, **kwargs)
    return updated_net


Instruction:
Make a summary of the following Python 3 code
def set_network_status(network_id,status,**kwargs):
    user_id = kwargs.get('user_id')
    #check_perm(user_id, 'delete_network')
    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
        net_i.check_write_permission(user_id)
        net_i.status = status
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))
    db.DBSession.flush()
    return 'OK'

Response: Activates a network by setting its status attribute to A.


Instruction:
Can you implement a function in Python 3 that
gives a network return its maximum extents.

Response:
def get_network_extents(network_id,**kwargs):
    """
    Given a network, return its maximum extents.
    This would be the minimum x value of all nodes,
    the minimum y value of all nodes,
    the maximum x value of all nodes and
    maximum y value of all nodes.

    @returns NetworkExtents object
    """
    rs = db.DBSession.query(Node.x, Node.y).filter(Node.network_id==network_id).all()
    if len(rs) == 0:
        return dict(
            network_id = network_id,
            min_x=None,
            max_x=None,
            min_y=None,
            max_y=None,
        )

    # Compute min/max extent of the network.
    x = [r.x for r in rs if r.x is not None]
    if len(x) > 0:
        x_min = min(x)
        x_max = max(x)
    else:
        # Default x extent if all None values
        x_min, x_max = 0, 1

    y = [r.y for r in rs if r.y is not None]
    if len(y) > 0:
        y_min = min(y)
        y_max = max(y)
    else:
        # Default y extent if all None values
        y_min, y_max = 0, 1

    ne = JSONObject(dict(
        network_id = network_id,
        min_x=x_min,
        max_x=x_max,
        min_y=y_min,
        max_y=y_max,
    ))
    return ne


Instruction:
Can you tell what is the following Python 3 function doing
def add_nodes(network_id, nodes,**kwargs):
    start_time = datetime.datetime.now()

    names=[]        # used to check uniqueness of node name
    for n_i in nodes:
        if n_i.name in names:
            raise HydraError("Duplicate Node Name: %s"%(n_i.name))
        names.append(n_i.name)

    user_id = kwargs.get('user_id')
    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
        net_i.check_write_permission(user_id)
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))

    _add_nodes_to_database(net_i, nodes)

    net_i.project_id=net_i.project_id
    db.DBSession.flush()

    node_s =  db.DBSession.query(Node).filter(Node.network_id==network_id).all()

    #Maps temporary node_ids to real node_ids
    node_id_map = dict()

    iface_nodes = dict()
    for n_i in node_s:
        iface_nodes[n_i.name] = n_i

    for node in nodes:
        node_id_map[node.id] = iface_nodes[node.name]

    _bulk_add_resource_attrs(network_id, 'NODE', nodes, iface_nodes)

    log.info("Nodes added in %s", get_timing(start_time))
    return node_s

Response: Add nodes to network


Instruction:
Here you have a function in Python 3, explain what it does
def add_links(network_id, links,**kwargs):
    '''
    add links to network
    '''
    start_time = datetime.datetime.now()
    user_id = kwargs.get('user_id')
    names=[]        # used to check uniqueness of link name before saving links to database
    for l_i in links:
        if l_i.name in names:
            raise HydraError("Duplicate Link Name: %s"%(l_i.name))
        names.append(l_i.name)

    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
        net_i.check_write_permission(user_id)
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))
    node_id_map=dict()
    for node in net_i.nodes:
       node_id_map[node.id]=node
    _add_links_to_database(net_i, links, node_id_map)

    net_i.project_id=net_i.project_id
    db.DBSession.flush()
    link_s =  db.DBSession.query(Link).filter(Link.network_id==network_id).all()
    iface_links = {}
    for l_i in link_s:
        iface_links[l_i.name] = l_i
    link_attrs = _bulk_add_resource_attrs(net_i.id, 'LINK', links, iface_links)
    log.info("Nodes added in %s", get_timing(start_time))
    return link_s

Response: add links to network


Instruction:
Can you generate a brief explanation for the following Python 3 code
def update_node(node, flush=True, **kwargs):
    user_id = kwargs.get('user_id')
    try:
        node_i = db.DBSession.query(Node).filter(Node.id == node.id).one()
    except NoResultFound:
        raise ResourceNotFoundError("Node %s not found"%(node.id))

    node_i.network.check_write_permission(user_id)

    node_i.name = node.name if node.name is not None else node_i.name
    node_i.x    = node.x if node.x is not None else node_i.x
    node_i.y    = node.y if node.y is not None else node_i.y
    node_i.description = node.description if node.description is not None else node_i.description
    node_i.layout      = node.get_layout() if node.layout is not None else node_i.layout

    if node.attributes is not None:
        _update_attributes(node_i, node.attributes)

    if node.types is not None:
        hdb.add_resource_types(node_i, node.types)

    if flush is True:
        db.DBSession.flush()

    return node_i

Response:
Update a node.
    If new attributes are present, they will be added to the node.
    The non-presence of attributes does not remove them.

    The flush argument indicates whether dbsession.flush should be called. THis
    is set to False when update_node is called from another function which does
    the flush.


Instruction:
Explain what the following Python 3 code does
def update_nodes(nodes,**kwargs):
    user_id = kwargs.get('user_id')
    updated_nodes = []
    for n in nodes:
        updated_node_i = update_node(n, flush=False, user_id=user_id)
        updated_nodes.append(updated_node_i)

    db.DBSession.flush()

    return updated_nodes

Response:
Update multiple nodes.
    If new attributes are present, they will be added to the node.
    The non-presence of attributes does not remove them.

    %TODO:merge this with the 'update_nodes' functionality in the 'update_netework'
    function, so we're not duplicating functionality. D.R.Y!

    returns: a list of updated nodes


Instruction:
How would you explain what the following Python 3 function does
def set_node_status(node_id, status, **kwargs):
    user_id = kwargs.get('user_id')
    try:
        node_i = db.DBSession.query(Node).filter(Node.id == node_id).one()
    except NoResultFound:
        raise ResourceNotFoundError("Node %s not found"%(node_id))

    node_i.network.check_write_permission(user_id)

    node_i.status = status

    for link in node_i.links_to:
        link.status = status
    for link in node_i.links_from:
        link.status = status

    db.DBSession.flush()

    return node_i

Response: Set the status of a node to X


Instruction:
Given the following Python 3 function, write the documentation
def purge_network(network_id, purge_data,**kwargs):
    user_id = kwargs.get('user_id')
    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))

    log.info("Deleting network %s, id=%s", net_i.name, network_id)

    net_i.check_write_permission(user_id)
    db.DBSession.delete(net_i)
    db.DBSession.flush()
    return 'OK'

Response: Remove a network from DB completely


Instruction:
How would you implement a function in Python 3 that
deletes all the datasets that are unique to a resource.

Response:
def _purge_datasets_unique_to_resource(ref_key, ref_id):
    """
        Find the number of times a a resource and dataset combination
        occurs. If this equals the number of times the dataset appears, then
        we can say this dataset is unique to this resource, therefore it can be deleted
    """
    count_qry = db.DBSession.query(ResourceScenario.dataset_id,
                                   func.count(ResourceScenario.dataset_id)).group_by(
                                       ResourceScenario.dataset_id).filter(
                                       ResourceScenario.resource_attr_id==ResourceAttr.id)

    if ref_key == 'NODE':
        count_qry.filter(ResourceAttr.node_id==ref_id)
    elif ref_key == 'LINK':
        count_qry.filter(ResourceAttr.link_id==ref_id)
    elif ref_key == 'GROUP':
        count_qry.filter(ResourceAttr.group_id==ref_id)

    count_rs = count_qry.all()

    for dataset_id, count in count_rs:
        full_dataset_count = db.DBSession.query(ResourceScenario).filter(ResourceScenario.dataset_id==dataset_id).count()
        if full_dataset_count == count:
            """First delete all the resource scenarios"""
            datasets_rs_to_delete = db.DBSession.query(ResourceScenario).filter(ResourceScenario.dataset_id==dataset_id).all()
            for dataset_rs in datasets_rs_to_delete:
                db.DBSession.delete(dataset_rs)

            """Then delete all the datasets"""
            dataset_to_delete = db.DBSession.query(Dataset).filter(Dataset.id==dataset_id).one()
            log.info("Deleting %s dataset %s (%s)", ref_key, dataset_to_delete.name, dataset_to_delete.id)
            db.DBSession.delete(dataset_to_delete)


Instruction:
Here you have a function in Python 3, explain what it does
def delete_node(node_id, purge_data,**kwargs):
    user_id = kwargs.get('user_id')
    try:
        node_i = db.DBSession.query(Node).filter(Node.id == node_id).one()
    except NoResultFound:
        raise ResourceNotFoundError("Node %s not found"%(node_id))

    group_items = db.DBSession.query(ResourceGroupItem).filter(
                                        ResourceGroupItem.node_id==node_id).all()
    for gi in group_items:
        db.DBSession.delete(gi)

    if purge_data == 'Y':
        _purge_datasets_unique_to_resource('NODE', node_id)

    log.info("Deleting node %s, id=%s", node_i.name, node_id)

    node_i.network.check_write_permission(user_id)
    db.DBSession.delete(node_i)
    db.DBSession.flush()
    return 'OK'

Response: Delete a node from DB completely


Instruction:
How would you implement a function in Python 3 that
adds a link to a network

Response:
def add_link(network_id, link,**kwargs):
    """
        Add a link to a network
    """
    user_id = kwargs.get('user_id')

    #check_perm(user_id, 'edit_topology')
    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
        net_i.check_write_permission(user_id)
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))

    try:
        node_1 = db.DBSession.query(Node).filter(Node.id==link.node_1_id).one()
        node_2 = db.DBSession.query(Node).filter(Node.id==link.node_2_id).one()
    except NoResultFound:
        raise ResourceNotFoundError("Nodes for link not found")

    link_i = net_i.add_link(link.name, link.description, link.layout, node_1, node_2)

    hdb.add_resource_attributes(link_i, link.attributes)

    db.DBSession.flush()

    if link.types is not None and len(link.types) > 0:
        res_types = []
        res_attrs = []
        res_scenarios = {}
        for typesummary in link.types:
            ra, rt, rs = template.set_resource_type(link_i,
                                        typesummary.id,
                                         **kwargs)
            res_types.append(rt)
            res_attrs.extend(ra)
            res_scenarios.update(rs)#rs is a dict

        if len(res_types) > 0:
            db.DBSession.bulk_insert_mappings(ResourceType, res_types)
        if len(res_attrs) > 0:
            db.DBSession.bulk_insert_mappings(ResourceAttr, res_attrs)

            new_res_attrs = db.DBSession.query(ResourceAttr).order_by(ResourceAttr.id.desc()).limit(len(res_attrs)).all()
            all_rs = []
            for ra in new_res_attrs:
                ra_id = ra.id
                if ra.attr_id in res_scenarios:
                    rs_list = res_scenarios[ra.attr_id]
                    for rs in rs_list:
                        rs_list[rs]['resource_attr_id'] = ra_id
                        all_rs.append(rs_list[rs])

            if len(all_rs) > 0:
                db.DBSession.bulk_insert_mappings(ResourceScenario, all_rs)

    db.DBSession.refresh(link_i)

    return link_i


Instruction:
Write a Python 3 script to
set the status of a link.

Response:
def set_link_status(link_id, status, **kwargs):
    """
        Set the status of a link
    """
    user_id = kwargs.get('user_id')
    #check_perm(user_id, 'edit_topology')
    try:
        link_i = db.DBSession.query(Link).filter(Link.id == link_id).one()
    except NoResultFound:
        raise ResourceNotFoundError("Link %s not found"%(link_id))

    link_i.network.check_write_permission(user_id)

    link_i.status = status
    db.DBSession.flush()


Instruction:
Can you generate the documentation for the following Python 3 function
def delete_link(link_id, purge_data,**kwargs):
    user_id = kwargs.get('user_id')
    try:
        link_i = db.DBSession.query(Link).filter(Link.id == link_id).one()
    except NoResultFound:
        raise ResourceNotFoundError("Link %s not found"%(link_id))

    group_items = db.DBSession.query(ResourceGroupItem).filter(
                                                    ResourceGroupItem.link_id==link_id).all()
    for gi in group_items:
        db.DBSession.delete(gi)

    if purge_data == 'Y':
        _purge_datasets_unique_to_resource('LINK', link_id)

    log.info("Deleting link %s, id=%s", link_i.name, link_id)

    link_i.network.check_write_permission(user_id)
    db.DBSession.delete(link_i)
    db.DBSession.flush()

Response: Delete a link from DB completely


Instruction:
Implement a Python 3 function for
adding a resourcegroup to a network

Response:
def add_group(network_id, group,**kwargs):
    """
        Add a resourcegroup to a network
    """

    user_id = kwargs.get('user_id')
    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
        net_i.check_write_permission(user_id=user_id)
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))

    res_grp_i = net_i.add_group(group.name, group.description, group.status)

    hdb.add_resource_attributes(res_grp_i, group.attributes)

    db.DBSession.flush()
    if group.types is not None and len(group.types) > 0:
        res_types = []
        res_attrs = []
        res_scenarios = {}
        for typesummary in group.types:
            ra, rt, rs = template.set_resource_type(res_grp_i,
                                        typesummary.id,
                                         **kwargs)
            res_types.append(rt)
            res_attrs.extend(ra)
            res_scenarios.update(rs)#rs is a dict
        if len(res_types) > 0:
            db.DBSession.bulk_insert_mappings(ResourceType, res_types)
        if len(res_attrs) > 0:
            db.DBSession.bulk_insert_mappings(ResourceAttr, res_attrs)

            new_res_attrs = db.DBSession.query(ResourceAttr).order_by(ResourceAttr.id.desc()).limit(len(res_attrs)).all()
            all_rs = []
            for ra in new_res_attrs:
                ra_id = ra.id
                if ra.attr_id in res_scenarios:
                    rs_list = res_scenarios[ra.attr_id]
                    for rs in rs_list:
                        rs_list[rs]['resource_attr_id'] = ra_id
                        all_rs.append(rs_list[rs])

            if len(all_rs) > 0:
                db.DBSession.bulk_insert_mappings(ResourceScenario, all_rs)

    db.DBSession.refresh(res_grp_i)

    return res_grp_i


Instruction:
Write a Python 3 script to
update a group. If new attributes are present, they will be added to the group. The non-presence of attributes does not remove them.

Response:
def update_group(group,**kwargs):
    """
        Update a group.
        If new attributes are present, they will be added to the group.
        The non-presence of attributes does not remove them.
    """
    user_id = kwargs.get('user_id')
    try:
        group_i = db.DBSession.query(ResourceGroup).filter(ResourceGroup.id == group.id).one()
    except NoResultFound:
        raise ResourceNotFoundError("group %s not found"%(group.id))

    group_i.network.check_write_permission(user_id)

    group_i.name = group.name if group.name != None else group_i.name
    group_i.description = group.description if group.description else group_i.description

    if group.attributes is not None:
        _update_attributes(group_i, group.attributes)

    if group.types is not None:
        hdb.add_resource_types(group_i, group.types)

    db.DBSession.flush()

    return group_i


Instruction:
Make a summary of the following Python 3 code
def set_group_status(group_id, status, **kwargs):
    user_id = kwargs.get('user_id')
    try:
        group_i = db.DBSession.query(ResourceGroup).filter(ResourceGroup.id == group_id).one()
    except NoResultFound:
        raise ResourceNotFoundError("ResourceGroup %s not found"%(group_id))

    group_i.network.check_write_permission(user_id)

    group_i.status = status

    db.DBSession.flush()

    return group_i

Response: Set the status of a group to X


Instruction:
Make a summary of the following Python 3 code
def delete_group(group_id, purge_data,**kwargs):
    user_id = kwargs.get('user_id')
    try:
        group_i = db.DBSession.query(ResourceGroup).filter(ResourceGroup.id == group_id).one()
    except NoResultFound:
        raise ResourceNotFoundError("Group %s not found"%(group_id))

    group_items = db.DBSession.query(ResourceGroupItem).filter(
                                                    ResourceGroupItem.group_id==group_id).all()
    for gi in group_items:
        db.DBSession.delete(gi)

    if purge_data == 'Y':
        _purge_datasets_unique_to_resource('GROUP', group_id)

    log.info("Deleting group %s, id=%s", group_i.name, group_id)

    group_i.network.check_write_permission(user_id)
    db.DBSession.delete(group_i)
    db.DBSession.flush()

Response: Delete group from DB completely


Instruction:
Can you generate the documentation for the following Python 3 function
def get_scenarios(network_id,**kwargs):

    user_id = kwargs.get('user_id')
    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
        net_i.check_read_permission(user_id=user_id)
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))

    return net_i.scenarios

Response: Get all the scenarios in a given network.


Instruction:
Here you have a function in Python 3, explain what it does
def validate_network_topology(network_id,**kwargs):

    user_id = kwargs.get('user_id')
    try:
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).one()
        net_i.check_write_permission(user_id=user_id)
    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))

    nodes = []
    for node_i in net_i.nodes:
        if node_i.status == 'A':
            nodes.append(node_i.node_id)

    link_nodes = []
    for link_i in net_i.links:
        if link_i.status != 'A':
            continue
        if link_i.node_1_id not in link_nodes:
            link_nodes.append(link_i.node_1_id)

        if link_i.node_2_id not in link_nodes:
            link_nodes.append(link_i.node_2_id)

    nodes = set(nodes)
    link_nodes = set(link_nodes)

    isolated_nodes = nodes - link_nodes

    return isolated_nodes

Response: Validate that the network topology is valid.


Instruction:
Here you have a function in Python 3, explain what it does
def get_resources_of_type(network_id, type_id, **kwargs):

    nodes_with_type = db.DBSession.query(Node).join(ResourceType).filter(Node.network_id==network_id, ResourceType.type_id==type_id).all()
    links_with_type = db.DBSession.query(Link).join(ResourceType).filter(Link.network_id==network_id, ResourceType.type_id==type_id).all()
    groups_with_type = db.DBSession.query(ResourceGroup).join(ResourceType).filter(ResourceGroup.network_id==network_id, ResourceType.type_id==type_id).all()

    return nodes_with_type, links_with_type, groups_with_type

Response: Get the nodes links and resource groups which have the type specified.


Instruction:
Here you have a function in Python 3, explain what it does
def clean_up_network(network_id, **kwargs):
    user_id = kwargs.get('user_id')
    #check_perm(user_id, 'delete_network')
    try:
        log.debug("Querying Network %s", network_id)
        net_i = db.DBSession.query(Network).filter(Network.id == network_id).\
        options(noload('scenarios')).options(noload('nodes')).options(noload('links')).options(noload('resourcegroups')).options(joinedload_all('types.templatetype.template')).one()
        net_i.attributes

        #Define the basic resource queries
        node_qry = db.DBSession.query(Node).filter(Node.network_id==network_id).filter(Node.status=='X').all()

        link_qry = db.DBSession.query(Link).filter(Link.network_id==network_id).filter(Link.status=='X').all()

        group_qry = db.DBSession.query(ResourceGroup).filter(ResourceGroup.network_id==network_id).filter(ResourceGroup.status=='X').all()

        scenario_qry = db.DBSession.query(Scenario).filter(Scenario.network_id==network_id).filter(Scenario.status=='X').all()

        for n in node_qry:
            db.DBSession.delete(n)
        for l in link_qry:
            db.DBSession.delete(l)
        for g in group_qry:
            db.DBSession.delete(g)
        for s in scenario_qry:
            db.DBSession.delete(s)

    except NoResultFound:
        raise ResourceNotFoundError("Network %s not found"%(network_id))
    db.DBSession.flush()
    return 'OK'

Response: Delete any deleted nodes links resourcegroups and scenarios in a given network


Instruction:
How would you code a function in Python 3 to
get all resource attributes in a network.

Response:
def get_all_resource_attributes_in_network(attr_id, network_id, **kwargs):
    """
        Find every resource attribute in the network matching the supplied attr_id
    """

    user_id = kwargs.get('user_id')

    try:
        a = db.DBSession.query(Attr).filter(Attr.id == attr_id).one()
    except NoResultFound:
        raise HydraError("Attribute %s not found"%(attr_id,))

    ra_qry = db.DBSession.query(ResourceAttr).filter(
                ResourceAttr.attr_id==attr_id,
                or_(Network.id == network_id,
                Node.network_id==network_id,
                Link.network_id==network_id,
                ResourceGroup.network_id==network_id)
            ).outerjoin('node')\
             .outerjoin('link')\
             .outerjoin('network')\
             .outerjoin('resourcegroup')\
             .options(joinedload_all('node'))\
             .options(joinedload_all('link'))\
             .options(joinedload_all('resourcegroup'))\
             .options(joinedload_all('network'))

    resourceattrs = ra_qry.all()

    json_ra = []
    #Load the metadata too
    for ra in resourceattrs:
        ra_j = JSONObject(ra, extras={'node':JSONObject(ra.node) if ra.node else None,
                                               'link':JSONObject(ra.link) if ra.link else None,
                                               'resourcegroup':JSONObject(ra.resourcegroup) if ra.resourcegroup else None,
                                               'network':JSONObject(ra.network) if ra.network else None})

        if ra_j.node is not None:
            ra_j.resource = ra_j.node
        elif ra_j.link is not None:
            ra_j.resource = ra_j.link
        elif ra_j.resourcegroup is not None:
            ra_j.resource = ra_j.resourcegroup
        elif ra.network is not None:
            ra_j.resource = ra_j.network

        json_ra.append(ra_j)

    return json_ra


Instruction:
Here you have a function in Python 3, explain what it does
def clone_network(network_id, recipient_user_id=None, new_network_name=None, project_id=None, project_name=None,  new_project=True, **kwargs):

    user_id = kwargs['user_id']

    ex_net = db.DBSession.query(Network).filter(Network.id==network_id).one()

    ex_net.check_read_permission(user_id)

    if project_id is None and new_project == True:

        log.info("Creating a new project for cloned network")

        ex_proj = db.DBSession.query(Project).filter(Project.id==ex_net.project_id).one()

        user = db.DBSession.query(User).filter(User.id==user_id).one()

        project = Project()
        if project_name is None or project_name=="":
            project_name=ex_proj.name + " (Cloned by %s)" % user.display_name

        #check a project with this name doesn't already exist:
        ex_project =  db.DBSession.query(Project).filter(Project.name==project_name,
                                                         Project.created_by==user_id).all()
        #If it exists, use it.
        if len(ex_project) > 0:
            project=ex_project[0]
        else:
            project.name = project_name
            project.created_by = user_id

            project.set_owner(user_id)

        if recipient_user_id!=None:
            project.set_owner(recipient_user_id)

        db.DBSession.add(project)
        db.DBSession.flush()

        project_id=project.id

    elif project_id is None:
        log.info("Using current project for cloned network")
        project_id=ex_net.project_id

    if new_network_name is None or new_network_name == "":
        new_network_name=ex_net.name

    log.info('Cloning Network...')

    #Find if there's any projects with this name in the project already
    ex_network =  db.DBSession.query(Network).filter(Network.project_id==project_id,
                                                     Network.name.like("{0}%".format(new_network_name))).all()

    if len(ex_network) > 0:
        new_network_name = new_network_name + " " + str(len(ex_network))

    newnet = Network()

    newnet.project_id = project_id
    newnet.name = new_network_name
    newnet.description = ex_net.description
    newnet.layout = ex_net.layout
    newnet.status = ex_net.status
    newnet.projection = ex_net.projection
    newnet.created_by = user_id

    newnet.set_owner(user_id)
    if recipient_user_id is not None:
        newnet.set_owner(recipient_user_id)

    db.DBSession.add(newnet)

    db.DBSession.flush()

    newnetworkid = newnet.id

    log.info('CLoning Nodes')
    node_id_map = _clone_nodes(network_id, newnetworkid)

    log.info('Cloning Links')
    link_id_map = _clone_links(network_id, newnetworkid, node_id_map)

    log.info('CLoning Groups')
    group_id_map = _clone_groups(network_id,
                                          newnetworkid,
                                          node_id_map,
                                          link_id_map)

    log.info("Cloning Resource Attributes")
    ra_id_map = _clone_resourceattrs(network_id, newnetworkid, node_id_map, link_id_map, group_id_map)

    log.info("Cloning Resource Types")
    _clone_resourcetypes(network_id, newnetworkid, node_id_map, link_id_map, group_id_map)

    log.info('Cloning Scenarios')
    _clone_scenarios(network_id, newnetworkid, ra_id_map, node_id_map, link_id_map, group_id_map, user_id)

    db.DBSession.flush()

    return newnetworkid

Response: Clone a network with the specified name and project.


Instruction:
Implement a function in Python 3 to
copy data from source to target scenario.

Response:
def copy_data_from_scenario(resource_attrs, source_scenario_id, target_scenario_id, **kwargs):
    """
        For a given list of resource attribute IDS copy the dataset_ids from
        the resource scenarios in the source scenario to those in the 'target' scenario.
    """

    #Get all the resource scenarios we wish to update
    target_resourcescenarios = db.DBSession.query(ResourceScenario).filter(
            ResourceScenario.scenario_id==target_scenario_id,
            ResourceScenario.resource_attr_id.in_(resource_attrs)).all()

    target_rs_dict = {}
    for target_rs in target_resourcescenarios:
        target_rs_dict[target_rs.resource_attr_id] = target_rs

    #get all the resource scenarios we are using to get our datsets source.
    source_resourcescenarios = db.DBSession.query(ResourceScenario).filter(
            ResourceScenario.scenario_id==source_scenario_id,
            ResourceScenario.resource_attr_id.in_(resource_attrs)).all()

    #If there is an RS in scenario 'source' but not in 'target', then create
    #a new one in 'target'
    for source_rs in source_resourcescenarios:
        target_rs = target_rs_dict.get(source_rs.resource_attr_id)
        if target_rs is not None:
            target_rs.dataset_id = source_rs.dataset_id
        else:
            target_rs = ResourceScenario()
            target_rs.scenario_id      = target_scenario_id
            target_rs.dataset_id       = source_rs.dataset_id
            target_rs.resource_attr_id = source_rs.resource_attr_id
            db.DBSession.add(target_rs)

    db.DBSession.flush()

    return target_resourcescenarios


Instruction:
How would you implement a function in Python 3 that
gets the specified scenario

Response:
def get_scenario(scenario_id,**kwargs):
    """
        Get the specified scenario
    """

    user_id = kwargs.get('user_id')

    scen_i = _get_scenario(scenario_id, user_id)

    scen_j = JSONObject(scen_i)
    rscen_rs = db.DBSession.query(ResourceScenario).filter(ResourceScenario.scenario_id==scenario_id).options(joinedload_all('dataset.metadata')).all()

    #lazy load resource attributes and attributes
    for rs in rscen_rs:
        rs.resourceattr
        rs.resourceattr.attr

    rgi_rs = db.DBSession.query(ResourceGroupItem).filter(ResourceGroupItem.scenario_id==scenario_id).all()

    scen_j.resourcescenarios = []
    for rs in rscen_rs:
        rs_j = JSONObject(rs, extras={'resourceattr':JSONObject(rs.resourceattr)})
        if rs.dataset.check_read_permission(user_id, do_raise=False) is False:
            rs_j.dataset['value'] = None
            rs_j.dataset.metadata = JSONObject({})
        scen_j.resourcescenarios.append(rs_j)

    scen_j.resourcegroupitems =[JSONObject(r) for r in rgi_rs]

    return scen_j


Instruction:
Can you tell what is the following Python 3 function doing
def add_scenario(network_id, scenario,**kwargs):
    user_id = int(kwargs.get('user_id'))
    log.info("Adding scenarios to network")

    _check_network_ownership(network_id, user_id)

    existing_scen = db.DBSession.query(Scenario).filter(Scenario.name==scenario.name, Scenario.network_id==network_id).first()
    if existing_scen is not None:
        raise HydraError("Scenario with name %s already exists in network %s"%(scenario.name, network_id))

    scen = Scenario()
    scen.name                 = scenario.name
    scen.description          = scenario.description
    scen.layout               = scenario.get_layout()
    scen.network_id           = network_id
    scen.created_by           = user_id
    scen.start_time           = str(timestamp_to_ordinal(scenario.start_time)) if scenario.start_time else None
    scen.end_time             = str(timestamp_to_ordinal(scenario.end_time)) if scenario.end_time else None
    scen.time_step            = scenario.time_step
    scen.resourcescenarios    = []
    scen.resourcegroupitems   = []

    #Just in case someone puts in a negative ID for the scenario.
    if scenario.id < 0:
        scenario.id = None

    if scenario.resourcescenarios is not None:
        #extract the data from each resourcescenario so it can all be
        #inserted in one go, rather than one at a time
        all_data = [r.dataset for r in scenario.resourcescenarios]

        datasets = data._bulk_insert_data(all_data, user_id=user_id)

        #record all the resource attribute ids
        resource_attr_ids = [r.resource_attr_id for r in scenario.resourcescenarios]

        #get all the resource scenarios into a list and bulk insert them
        for i, ra_id in enumerate(resource_attr_ids):
            rs_i = ResourceScenario()
            rs_i.resource_attr_id = ra_id
            rs_i.dataset_id       = datasets[i].id
            rs_i.scenario_id      = scen.id
            rs_i.dataset = datasets[i]
            scen.resourcescenarios.append(rs_i)

    if scenario.resourcegroupitems is not None:
        #Again doing bulk insert.
        for group_item in scenario.resourcegroupitems:
            group_item_i = ResourceGroupItem()
            group_item_i.scenario_id = scen.id
            group_item_i.group_id    = group_item.group_id
            group_item_i.ref_key     = group_item.ref_key
            if group_item.ref_key == 'NODE':
                group_item_i.node_id      = group_item.ref_id
            elif group_item.ref_key == 'LINK':
                group_item_i.link_id      = group_item.ref_id
            elif group_item.ref_key == 'GROUP':
                group_item_i.subgroup_id  = group_item.ref_id
            scen.resourcegroupitems.append(group_item_i)
    db.DBSession.add(scen)
    db.DBSession.flush()
    return scen

Response: Add a scenario to a network.


Instruction:
Write a Python 3 script to
update a single scenario in the DB.

Response:
def update_scenario(scenario,update_data=True,update_groups=True,flush=True,**kwargs):
    """
        Update a single scenario
        as all resources already exist, there is no need to worry
        about negative IDS

        flush = True flushes to the DB at the end of the function.
        flush = False does not flush, assuming that it will happen as part
                of another process, like update_network.
    """
    user_id = kwargs.get('user_id')
    scen = _get_scenario(scenario.id, user_id)

    if scen.locked == 'Y':
        raise PermissionError('Scenario is locked. Unlock before editing.')

    start_time = None
    if isinstance(scenario.start_time, float):
        start_time = six.text_type(scenario.start_time)
    else:
        start_time = timestamp_to_ordinal(scenario.start_time)
        if start_time is not None:
            start_time = six.text_type(start_time)

    end_time = None
    if isinstance(scenario.end_time, float):
        end_time = six.text_type(scenario.end_time)
    else:
        end_time = timestamp_to_ordinal(scenario.end_time)
        if end_time is not None:
            end_time = six.text_type(end_time)

    scen.name                 = scenario.name
    scen.description          = scenario.description
    scen.layout               = scenario.get_layout()
    scen.start_time           = start_time
    scen.end_time             = end_time
    scen.time_step            = scenario.time_step

    if scenario.resourcescenarios == None:
        scenario.resourcescenarios = []
    if scenario.resourcegroupitems == None:
        scenario.resourcegroupitems = []

    #lazy load resourcescenarios from the DB
    scen.resourcescenarios

    if update_data is True:
        datasets = [rs.dataset for rs in scenario.resourcescenarios]
        updated_datasets = data._bulk_insert_data(datasets, user_id, kwargs.get('app_name'))
        for i, r_scen in enumerate(scenario.resourcescenarios):
            _update_resourcescenario(scen, r_scen, dataset=updated_datasets[i], user_id=user_id, source=kwargs.get('app_name'))

    #lazy load resource grou items from the DB
    scen.resourcegroupitems

    if update_groups is True:
        #Get all the exiting resource group items for this scenario.
        #THen process all the items sent to this handler.
        #Any in the DB that are not passed in here are removed.
        for group_item in scenario.resourcegroupitems:
            _add_resourcegroupitem(group_item, scenario.id)

    if flush is True:
        db.DBSession.flush()

    return scen


Instruction:
Here you have a function in Python 3, explain what it does
def set_scenario_status(scenario_id, status, **kwargs):

    user_id = kwargs.get('user_id')

    _check_can_edit_scenario(scenario_id, kwargs['user_id'])

    scenario_i = _get_scenario(scenario_id, user_id)

    scenario_i.status = status
    db.DBSession.flush()
    return 'OK'

Response: Set the status of a scenario.


Instruction:
Write a Python 3 script to
set the status of a scenario.

Response:
def purge_scenario(scenario_id, **kwargs):
    """
        Set the status of a scenario.
    """

    _check_can_edit_scenario(scenario_id, kwargs['user_id'])

    user_id = kwargs.get('user_id')

    scenario_i = _get_scenario(scenario_id, user_id)

    db.DBSession.delete(scenario_i)
    db.DBSession.flush()
    return 'OK'


Instruction:
Write a Python 3 function that can
turn a dictionary into a named tuple so it can be passed into the constructor of a complex model generator.

Response:
def _get_as_obj(obj_dict, name):
    """
        Turn a dictionary into a named tuple so it can be
        passed into the constructor of a complex model generator.
    """
    if obj_dict.get('_sa_instance_state'):
        del obj_dict['_sa_instance_state']
    obj = namedtuple(name, tuple(obj_dict.keys()))
    for k, v in obj_dict.items():
        setattr(obj, k, v)
        log.info("%s = %s",k,getattr(obj,k))
    return obj


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_resource_scenario(resource_attr_id, scenario_id, **kwargs):
    user_id = kwargs.get('user_id')

    _get_scenario(scenario_id, user_id)

    try:
        rs = db.DBSession.query(ResourceScenario).filter(ResourceScenario.resource_attr_id==resource_attr_id,
                                             ResourceScenario.scenario_id == scenario_id).options(joinedload_all('dataset')).options(joinedload_all('dataset.metadata')).one()

        return rs
    except NoResultFound:
        raise ResourceNotFoundError("resource scenario for %s not found in scenario %s"%(resource_attr_id, scenario_id))

Response: Get the resource scenario object for a given resource attribute and scenario.


Instruction:
Here you have a function in Python 3, explain what it does
def bulk_update_resourcedata(scenario_ids, resource_scenarios,**kwargs):
    user_id = kwargs.get('user_id')
    res = None

    res = {}

    net_ids = db.DBSession.query(Scenario.network_id).filter(Scenario.id.in_(scenario_ids)).all()

    if len(set(net_ids)) != 1:
        raise HydraError("Scenario IDS are not in the same network")

    for scenario_id in scenario_ids:
        _check_can_edit_scenario(scenario_id, kwargs['user_id'])

        scen_i = _get_scenario(scenario_id, user_id)
        res[scenario_id] = []

        for rs in resource_scenarios:
            if rs.dataset is not None:
                updated_rs = _update_resourcescenario(scen_i, rs, user_id=user_id, source=kwargs.get('app_name'))
                res[scenario_id].append(updated_rs)
            else:
                _delete_resourcescenario(scenario_id, rs.resource_attr_id)

        db.DBSession.flush()

    return res

Response: Bulk update the data associated with a list of scenarios.


Instruction:
Can you generate the documentation for the following Python 3 function
def update_resourcedata(scenario_id, resource_scenarios,**kwargs):
    user_id = kwargs.get('user_id')
    res = None

    _check_can_edit_scenario(scenario_id, kwargs['user_id'])

    scen_i = _get_scenario(scenario_id, user_id)

    res = []
    for rs in resource_scenarios:
        if rs.dataset is not None:
            updated_rs = _update_resourcescenario(scen_i, rs, user_id=user_id, source=kwargs.get('app_name'))
            res.append(updated_rs)
        else:
            _delete_resourcescenario(scenario_id, rs.resource_attr_id)

    db.DBSession.flush()

    return res

Response: Update the data associated with a resource scenario.


Instruction:
Can you create a Python 3 function that
deletes the data associated with a resource in a scenario.

Response:
def delete_resource_scenario(scenario_id, resource_attr_id, quiet=False, **kwargs):
    """
        Remove the data associated with a resource in a scenario.
    """
    _check_can_edit_scenario(scenario_id, kwargs['user_id'])

    _delete_resourcescenario(scenario_id, resource_attr_id, suppress_error=quiet)


Instruction:
Here you have a function in Python 3, explain what it does
def delete_resourcedata(scenario_id, resource_scenario, quiet = False, **kwargs):

    _check_can_edit_scenario(scenario_id, kwargs['user_id'])

    _delete_resourcescenario(scenario_id, resource_scenario.resource_attr_id, suppress_error=quiet)

Response: Delete the data associated with a resource in a scenario.


Instruction:
How would you code a function in Python 3 to
update the value of a resource attribute in a scenario.

Response:
def _update_resourcescenario(scenario, resource_scenario, dataset=None, new=False, user_id=None, source=None):
    """
        Insert or Update the value of a resource's attribute by first getting the
        resource, then parsing the input data, then assigning the value.

        returns a ResourceScenario object.
    """
    if scenario is None:
        scenario = db.DBSession.query(Scenario).filter(Scenario.id==1).one()

    ra_id = resource_scenario.resource_attr_id

    log.debug("Assigning resource attribute: %s",ra_id)
    try:
        r_scen_i = db.DBSession.query(ResourceScenario).filter(
                        ResourceScenario.scenario_id==scenario.id,
                        ResourceScenario.resource_attr_id==ra_id).one()
    except NoResultFound as e:
        log.info("Creating new RS")
        r_scen_i = ResourceScenario()
        r_scen_i.resource_attr_id = resource_scenario.resource_attr_id
        r_scen_i.scenario_id      = scenario.id
        r_scen_i.scenario = scenario

        db.DBSession.add(r_scen_i)

    if scenario.locked == 'Y':
        log.info("Scenario %s is locked",scenario.id)
        return r_scen_i

    if dataset is not None:
        r_scen_i.dataset = dataset

        return r_scen_i

    dataset = resource_scenario.dataset

    value = dataset.parse_value()

    log.info("Assigning %s to resource attribute: %s", value, ra_id)

    if value is None:
        log.info("Cannot set data on resource attribute %s",ra_id)
        return None

    metadata = dataset.get_metadata_as_dict(source=source, user_id=user_id)
    data_unit_id = dataset.unit_id

    data_hash = dataset.get_hash(value, metadata)

    assign_value(r_scen_i,
                 dataset.type.lower(),
                 value,
                 data_unit_id,
                 dataset.name,
                 metadata=metadata,
                 data_hash=data_hash,
                 user_id=user_id,
                 source=source)
    return r_scen_i