Instruction:
Make a summary of the following Python 3 code
def request_time_facet(field, time_filter, time_gap, time_limit=100):
    start, end = parse_datetime_range(time_filter)

    key_range_start = "f.{0}.facet.range.start".format(field)
    key_range_end = "f.{0}.facet.range.end".format(field)
    key_range_gap = "f.{0}.facet.range.gap".format(field)
    key_range_mincount = "f.{0}.facet.mincount".format(field)

    if time_gap:
        gap = gap_to_sorl(time_gap)
    else:
        gap = compute_gap(start, end, time_limit)

    value_range_start = start.get("parsed_datetime")
    if start.get("is_common_era"):
        value_range_start = start.get("parsed_datetime").isoformat().replace("+00:00", "") + "Z"

    value_range_end = start.get("parsed_datetime")
    if end.get("is_common_era"):
        value_range_end = end.get("parsed_datetime").isoformat().replace("+00:00", "") + "Z"

    value_range_gap = gap

    params = {
        'facet.range': field,
        key_range_start: value_range_start,
        key_range_end: value_range_end,
        key_range_gap: value_range_gap,
        key_range_mincount: 1
    }

    return params

Response: Request a time facet query builder for a given field.


Instruction:
Create a Python 3 function for
parsing Solr Geo Range as Pair

Response:
def parse_solr_geo_range_as_pair(geo_box_str):
    """
    :param geo_box_str: [-90,-180 TO 90,180]
    :return: ("-90,-180", "90,180")
    """
    pattern = "\\[(.*) TO (.*)\\]"
    matcher = re.search(pattern, geo_box_str)
    if matcher:
        return matcher.group(1), matcher.group(2)
    else:
        raise Exception("Regex {0} could not parse {1}".format(pattern, geo_box_str))


Instruction:
How would you code a function in Python 3 to
parse a shapely. geometry. box into shapely. geometry. box

Response:
def parse_geo_box(geo_box_str):
    """
    parses [-90,-180 TO 90,180] to a shapely.geometry.box
    :param geo_box_str:
    :return:
    """

    from_point_str, to_point_str = parse_solr_geo_range_as_pair(geo_box_str)
    from_point = parse_lat_lon(from_point_str)
    to_point = parse_lat_lon(to_point_str)
    rectangle = box(from_point[0], from_point[1], to_point[0], to_point[1])
    return rectangle


Instruction:
Can you generate the documentation for the following Python 3 function
def request_heatmap_facet(field, hm_filter, hm_grid_level, hm_limit):

    if not hm_filter:
        hm_filter = '[-90,-180 TO 90,180]'

    params = {
        'facet': 'on',
        'facet.heatmap': field,
        'facet.heatmap.geom': hm_filter
    }

    if hm_grid_level:
        # note: aHmLimit is ignored in this case
        params['facet.heatmap.gridLevel'] = hm_grid_level
    else:
        # Calculate distErr that will approximate aHmLimit many cells as an upper bound
        rectangle = parse_geo_box(hm_filter)
        degrees_side_length = rectangle.length / 2
        cell_side_length = math.sqrt(float(hm_limit))
        cell_side_length_degrees = degrees_side_length / cell_side_length * 2
        params['facet.heatmap.distErr'] = str(float(cell_side_length_degrees))
        # TODO: not sure about if returning correct param values.

    # get_params = urllib.urlencode(params)
    return params

Response: Request a heatmap facet query builder for a given field.


Instruction:
How would you code a function in Python 3 to
translate asterisk to min and max

Response:
def asterisk_to_min_max(field, time_filter, search_engine_endpoint, actual_params=None):
    """
    traduce [* TO *] to something like [MIN-INDEXED-DATE TO MAX-INDEXED-DATE]
    :param field: map the stats to this field.
    :param time_filter: this is the value to be translated. think in "[* TO 2000]"
    :param search_engine_endpoint: solr core
    :param actual_params: (not implemented) to merge with other params.
    :return: translated time filter
    """

    if actual_params:
        raise NotImplemented("actual_params")

    start, end = parse_solr_time_range_as_pair(time_filter)
    if start == '*' or end == '*':
        params_stats = {
            "q": "*:*",
            "rows": 0,
            "stats.field": field,
            "stats": "true",
            "wt": "json"
        }
        res_stats = requests.get(search_engine_endpoint, params=params_stats)

        if res_stats.ok:

            stats_date_field = res_stats.json()["stats"]["stats_fields"][field]
            date_min = stats_date_field["min"]
            date_max = stats_date_field["max"]

            if start != '*':
                date_min = start
            if end != '*':
                date_max = end

            time_filter = "[{0} TO {1}]".format(date_min, date_max)

    return time_filter


Instruction:
Can you tell what is the following Python 3 function doing
def get_service(raw_xml):
    from pycsw.core.etree import etree

    parsed = etree.fromstring(raw_xml, etree.XMLParser(resolve_entities=False))

    # <dc:format>OGC:WMS</dc:format>
    source_tag = parsed.find("{http://purl.org/dc/elements/1.1/}source")
    # <dc:source>
    #    http://ngamaps.geointapps.org/arcgis/services/RIO/Rio_Foundation_Transportation/MapServer/WMSServer
    # </dc:source>
    format_tag = parsed.find("{http://purl.org/dc/elements/1.1/}format")

    service_url = None
    service_type = None

    if hasattr(source_tag, 'text'):
        service_url = source_tag.text

    if hasattr(format_tag, 'text'):
        service_type = format_tag.text

    if hasattr(format_tag, 'text'):
        service_type = format_tag.text

    service, created = Service.objects.get_or_create(url=service_url,
                                                     is_monitored=False,
                                                     type=service_type)
    # TODO: dont hardcode SRS, get them from the parsed XML.
    srs, created = SpatialReferenceSystem.objects.get_or_create(code="EPSG:4326")
    service.srs.add(srs)

    return service

Response: Get a service object based on the XML metadata


Instruction:
Can you implement a function in Python 3 that
queries by list of identifiers

Response:
def query_ids(self, ids):
        """
        Query by list of identifiers
        """

        results = self._get_repo_filter(Layer.objects).filter(uuid__in=ids).all()

        if len(results) == 0:  # try services
            results = self._get_repo_filter(Service.objects).filter(uuid__in=ids).all()

        return results


Instruction:
Write a Python 3 script for
querying by property domain values

Response:
def query_domain(self, domain, typenames, domainquerytype='list', count=False):
        """
        Query by property domain values
        """

        objects = self._get_repo_filter(Layer.objects)

        if domainquerytype == 'range':
            return [tuple(objects.aggregate(Min(domain), Max(domain)).values())]
        else:
            if count:
                return [(d[domain], d['%s__count' % domain])
                        for d in objects.values(domain).annotate(Count(domain))]
            else:
                return objects.values_list(domain).distinct()


Instruction:
Implement a function in Python 3 to
query to get latest or earliest update to repository

Response:
def query_insert(self, direction='max'):
        """
        Query to get latest (default) or earliest update to repository
        """
        if direction == 'min':
            return Layer.objects.aggregate(
                Min('last_updated'))['last_updated__min'].strftime('%Y-%m-%dT%H:%M:%SZ')
        return self._get_repo_filter(Layer.objects).aggregate(
            Max('last_updated'))['last_updated__max'].strftime('%Y-%m-%dT%H:%M:%SZ')


Instruction:
Here you have a function in Python 3, explain what it does
def query(self, constraint, sortby=None, typenames=None, maxrecords=10, startposition=0):

        # run the raw query and get total
        # we want to exclude layers which are not valid, as it is done in the search engine
        if 'where' in constraint:  # GetRecords with constraint
            query = self._get_repo_filter(Layer.objects).filter(
                is_valid=True).extra(where=[constraint['where']], params=constraint['values'])
        else:  # GetRecords sans constraint
            query = self._get_repo_filter(Layer.objects).filter(is_valid=True)

        total = query.count()

        # apply sorting, limit and offset
        if sortby is not None:
            if 'spatial' in sortby and sortby['spatial']:  # spatial sort
                desc = False
                if sortby['order'] == 'DESC':
                    desc = True
                query = query.all()
                return [str(total),
                        sorted(query,
                               key=lambda x: float(util.get_geometry_area(getattr(x, sortby['propertyname']))),
                               reverse=desc,
                               )[startposition:startposition+int(maxrecords)]]
            else:
                if sortby['order'] == 'DESC':
                    pname = '-%s' % sortby['propertyname']
                else:
                    pname = sortby['propertyname']
                return [str(total),
                        query.order_by(pname)[startposition:startposition+int(maxrecords)]]
        else:  # no sort
            return [str(total), query.all()[startposition:startposition+int(maxrecords)]]

Response: Query records from underlying repository


Instruction:
Can you generate a brief explanation for the following Python 3 code
def insert(self, resourcetype, source, insert_date=None):

        caller = inspect.stack()[1][3]

        if caller == 'transaction':  # insert of Layer
            hhclass = 'Layer'
            source = resourcetype
            resourcetype = resourcetype.csw_schema
        else:  # insert of service
            hhclass = 'Service'
            if resourcetype not in HYPERMAP_SERVICE_TYPES.keys():
                raise RuntimeError('Unsupported Service Type')

        return self._insert_or_update(resourcetype, source, mode='insert', hhclass=hhclass)

Response: Insert a record into the repository.


Instruction:
Can you write a function in Python 3 where it
inserts or update a record in the repository

Response:
def _insert_or_update(self, resourcetype, source, mode='insert', hhclass='Service'):
        """
        Insert or update a record in the repository
        """

        keywords = []

        if self.filter is not None:
            catalog = Catalog.objects.get(id=int(self.filter.split()[-1]))

        try:
            if hhclass == 'Layer':
                # TODO: better way of figuring out duplicates
                match = Layer.objects.filter(name=source.name,
                                             title=source.title,
                                             abstract=source.abstract,
                                             is_monitored=False)
                matches = match.all()
                if matches:
                    if mode == 'insert':
                        raise RuntimeError('HHypermap error: Layer %d \'%s\' already exists' % (
                                           matches[0].id, source.title))
                    elif mode == 'update':
                        match.update(
                            name=source.name,
                            title=source.title,
                            abstract=source.abstract,
                            is_monitored=False,
                            xml=source.xml,
                            wkt_geometry=source.wkt_geometry,
                            anytext=util.get_anytext([source.title, source.abstract, source.keywords_csv])
                        )

                service = get_service(source.xml)
                res, keywords = create_layer_from_metadata_xml(resourcetype, source.xml,
                                                               monitor=False, service=service,
                                                               catalog=catalog)

                res.save()

                LOGGER.debug('Indexing layer with id %s on search engine' % res.uuid)
                index_layer(res.id, use_cache=True)

            else:
                if resourcetype == 'http://www.opengis.net/cat/csw/2.0.2':
                    res = Endpoint(url=source, catalog=catalog)
                else:
                    res = Service(type=HYPERMAP_SERVICE_TYPES[resourcetype], url=source, catalog=catalog)

                res.save()

            if keywords:
                for kw in keywords:
                    res.keywords.add(kw)
        except Exception as err:
            raise RuntimeError('HHypermap error: %s' % err)

        # return a list of ids that were inserted or updated
        ids = []

        if hhclass == 'Layer':
            ids.append({'identifier': res.uuid, 'title': res.title})
        else:
            if resourcetype == 'http://www.opengis.net/cat/csw/2.0.2':
                for res in Endpoint.objects.filter(url=source).all():
                    ids.append({'identifier': res.uuid, 'title': res.url})
            else:
                for res in Service.objects.filter(url=source).all():
                    ids.append({'identifier': res.uuid, 'title': res.title})

        return ids


Instruction:
Can you generate a brief explanation for the following Python 3 code
def delete(self, constraint):

        results = self._get_repo_filter(Service.objects).extra(where=[constraint['where']],
                                                               params=constraint['values']).all()
        deleted = len(results)
        results.delete()
        return deleted

Response: Delete a record from the repository


Instruction:
Can you tell what is the following Python 3 function doing
def _get_repo_filter(self, query):
        if self.filter is not None:
            return query.extra(where=[self.filter])
        return query

Response: Apply repository wide side filter


Instruction:
Can you generate a brief explanation for the following Python 3 code
def to_meshlevel(meshcode):

    length = len(str(meshcode))
    if length == 4:
        return 1

    if length == 5:
        return 40000

    if length == 6:
        return 2

    if length == 7:
        if meshcode[6:7] in ['1','2','3','4']:
            return 5000

        if meshcode[6:7] == '6':
            return 8000

        if meshcode[6:7] == '5':
            return 20000

        if meshcode[6:7] == '7':
            return 16000

    if length == 8:
        return 3

    if length == 9:
        if meshcode[8:9] in ['1','2','3','4']:
            return 4

        if meshcode[8:9] == '5':
            return 2000

        if meshcode[8:9] == '6':
            return 2500

        if meshcode[8:9] == '7':
            return 4000

    if length == 10:
        if meshcode[9:10] in ['1','2','3','4']:
            return 5

    if length == 11:
        if meshcode[10:11] in ['1','2','3','4']:
            return 6

    raise ValueError('the meshcode is unsupported.')

Response: Returns the number of nodes in a given meshcode.


Instruction:
Here you have a function in Python 3, explain what it does
def to_meshpoint(meshcode, lat_multiplier, lon_multiplier):

    def mesh_cord(func_higher_cord, func_unit_cord, func_multiplier):
        return func_higher_cord() + func_unit_cord() * func_multiplier()

    lat_multiplier_lv = lambda: lat_multiplier

    lon_multiplier_lv = lambda: lon_multiplier

    lat_multiplier_lv1 = _functools.partial(
        lambda meshcode: int(meshcode[0:2]), meshcode=meshcode)

    lon_multiplier_lv1 = _functools.partial(
        lambda meshcode: int(meshcode[2:4]), meshcode=meshcode)

    lat_multiplier_40000 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[4:5])-1)[2:].zfill(2)[0:1]), meshcode=meshcode)

    lon_multiplier_40000 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[4:5])-1)[2:].zfill(2)[1:2]), meshcode=meshcode)

    lat_multiplier_20000 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[5:6])-1)[2:].zfill(2)[0:1]), meshcode=meshcode)

    lon_multiplier_20000 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[5:6])-1)[2:].zfill(2)[1:2]), meshcode=meshcode)

    lat_multiplier_16000 = _functools.partial(
        lambda meshcode: int(meshcode[4:5])/2, meshcode=meshcode)

    lon_multiplier_16000 = _functools.partial(
        lambda meshcode: int(meshcode[5:6])/2, meshcode=meshcode)

    lat_multiplier_lv2 = _functools.partial(
        lambda meshcode: int(meshcode[4:5]), meshcode=meshcode)

    lon_multiplier_lv2 = _functools.partial(
        lambda meshcode: int(meshcode[5:6]), meshcode=meshcode)

    lat_multiplier_8000 = _functools.partial(
        lambda meshcode: int(meshcode[4:5]), meshcode=meshcode)

    lon_multiplier_8000 = _functools.partial(
        lambda meshcode: int(meshcode[5:6]), meshcode=meshcode)

    lat_multiplier_5000 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[6:7])-1)[2:].zfill(2)[0:1]), meshcode=meshcode)

    lon_multiplier_5000 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[6:7])-1)[2:].zfill(2)[1:2]), meshcode=meshcode)

    lat_multiplier_4000 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[7:8])-1)[2:].zfill(2)[0:1]), meshcode=meshcode)

    lon_multiplier_4000 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[7:8])-1)[2:].zfill(2)[1:2]), meshcode=meshcode)

    lat_multiplier_2500 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[7:8])-1)[2:].zfill(2)[0:1]), meshcode=meshcode)

    lon_multiplier_2500 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[7:8])-1)[2:].zfill(2)[1:2]), meshcode=meshcode)

    lat_multiplier_2000 = _functools.partial(
        lambda meshcode: int(meshcode[6:7])/2, meshcode=meshcode)

    lon_multiplier_2000 = _functools.partial(
        lambda meshcode: int(meshcode[7:8])/2, meshcode=meshcode)

    lat_multiplier_lv3 = _functools.partial(
        lambda meshcode: int(meshcode[6:7]), meshcode=meshcode)

    lon_multiplier_lv3 = _functools.partial(
        lambda meshcode: int(meshcode[7:8]), meshcode=meshcode)

    lat_multiplier_lv4 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[8:9])-1)[2:].zfill(2)[0:1]), meshcode=meshcode)

    lon_multiplier_lv4 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[8:9])-1)[2:].zfill(2)[1:2]), meshcode=meshcode)

    lat_multiplier_lv5 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[9:10])-1)[2:].zfill(2)[0:1]), meshcode=meshcode)

    lon_multiplier_lv5 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[9:10])-1)[2:].zfill(2)[1:2]), meshcode=meshcode)

    lat_multiplier_lv6 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[10:11])-1)[2:].zfill(2)[0:1]), meshcode=meshcode)

    lon_multiplier_lv6 = _functools.partial(
        lambda meshcode: int(bin(int(meshcode[10:11])-1)[2:].zfill(2)[1:2]), meshcode=meshcode)

    mesh_lv1_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=lambda: 0,
        func_unit_cord=_unit_lat_lv1,
        func_multiplier=lat_multiplier_lv1)

    mesh_lv1_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=lambda: 100,
        func_unit_cord=_unit_lon_lv1,
        func_multiplier=lon_multiplier_lv1)

    mesh_40000_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv1_default_lat,
        func_unit_cord=_unit_lat_40000,
        func_multiplier=lat_multiplier_40000)

    mesh_40000_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv1_default_lon,
        func_unit_cord=_unit_lon_40000,
        func_multiplier=lon_multiplier_40000)

    mesh_20000_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_40000_default_lat,
        func_unit_cord=_unit_lat_20000,
        func_multiplier=lat_multiplier_20000)

    mesh_20000_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_40000_default_lon,
        func_unit_cord=_unit_lon_20000,
        func_multiplier=lon_multiplier_20000)

    mesh_16000_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv1_default_lat,
        func_unit_cord=_unit_lat_16000,
        func_multiplier=lat_multiplier_16000)

    mesh_16000_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv1_default_lon,
        func_unit_cord=_unit_lon_16000,
        func_multiplier=lon_multiplier_16000)

    mesh_lv2_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv1_default_lat,
        func_unit_cord=_unit_lat_lv2,
        func_multiplier=lat_multiplier_lv2)

    mesh_lv2_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv1_default_lon,
        func_unit_cord=_unit_lon_lv2,
        func_multiplier=lon_multiplier_lv2)

    mesh_8000_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv1_default_lat,
        func_unit_cord=_unit_lat_8000,
        func_multiplier=lat_multiplier_8000)

    mesh_8000_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv1_default_lon,
        func_unit_cord=_unit_lon_8000,
        func_multiplier=lon_multiplier_8000)

    mesh_5000_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv2_default_lat,
        func_unit_cord=_unit_lat_5000,
        func_multiplier=lat_multiplier_5000)

    mesh_5000_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv2_default_lon,
        func_unit_cord=_unit_lon_5000,
        func_multiplier=lon_multiplier_5000)

    mesh_4000_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_8000_default_lat,
        func_unit_cord=_unit_lat_4000,
        func_multiplier=lat_multiplier_4000)

    mesh_4000_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_8000_default_lon,
        func_unit_cord=_unit_lon_4000,
        func_multiplier=lon_multiplier_4000)

    mesh_2500_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_5000_default_lat,
        func_unit_cord=_unit_lat_2500,
        func_multiplier=lat_multiplier_2500)

    mesh_2500_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_5000_default_lon,
        func_unit_cord=_unit_lon_2500,
        func_multiplier=lon_multiplier_2500)

    mesh_2000_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv2_default_lat,
        func_unit_cord=_unit_lat_2000,
        func_multiplier=lat_multiplier_2000)

    mesh_2000_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv2_default_lon,
        func_unit_cord=_unit_lon_2000,
        func_multiplier=lon_multiplier_2000)

    mesh_lv3_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv2_default_lat,
        func_unit_cord=_unit_lat_lv3,
        func_multiplier=lat_multiplier_lv3)

    mesh_lv3_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv2_default_lon,
        func_unit_cord=_unit_lon_lv3,
        func_multiplier=lon_multiplier_lv3)

    mesh_lv4_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv3_default_lat,
        func_unit_cord=_unit_lat_lv4,
        func_multiplier=lat_multiplier_lv4)

    mesh_lv4_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv3_default_lon,
        func_unit_cord=_unit_lon_lv4,
        func_multiplier=lon_multiplier_lv4)

    mesh_lv5_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv4_default_lat,
        func_unit_cord=_unit_lat_lv5,
        func_multiplier=lat_multiplier_lv5)

    mesh_lv5_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv4_default_lon,
        func_unit_cord=_unit_lon_lv5,
        func_multiplier=lon_multiplier_lv5)

    mesh_lv6_default_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv5_default_lat,
        func_unit_cord=_unit_lat_lv6,
        func_multiplier=lat_multiplier_lv6)

    mesh_lv6_default_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv5_default_lon,
        func_unit_cord=_unit_lon_lv6,
        func_multiplier=lon_multiplier_lv6)

    mesh_lv1_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv1_default_lat,
        func_unit_cord=_unit_lat_lv1,
        func_multiplier=lat_multiplier_lv)

    mesh_lv1_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv1_default_lon,
        func_unit_cord=_unit_lon_lv1,
        func_multiplier=lon_multiplier_lv)

    mesh_40000_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_40000_default_lat,
        func_unit_cord=_unit_lat_40000,
        func_multiplier=lat_multiplier_lv)

    mesh_40000_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_40000_default_lon,
        func_unit_cord=_unit_lon_40000,
        func_multiplier=lon_multiplier_lv)

    mesh_20000_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_20000_default_lat,
        func_unit_cord=_unit_lat_20000,
        func_multiplier=lat_multiplier_lv)

    mesh_20000_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_20000_default_lon,
        func_unit_cord=_unit_lon_20000,
        func_multiplier=lon_multiplier_lv)

    mesh_16000_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_16000_default_lat,
        func_unit_cord=_unit_lat_16000,
        func_multiplier=lat_multiplier_lv)

    mesh_16000_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_16000_default_lon,
        func_unit_cord=_unit_lon_16000,
        func_multiplier=lon_multiplier_lv)

    mesh_lv2_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv2_default_lat,
        func_unit_cord=_unit_lat_lv2,
        func_multiplier=lat_multiplier_lv)

    mesh_lv2_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv2_default_lon,
        func_unit_cord=_unit_lon_lv2,
        func_multiplier=lon_multiplier_lv)

    mesh_8000_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_8000_default_lat,
        func_unit_cord=_unit_lat_8000,
        func_multiplier=lat_multiplier_lv)

    mesh_8000_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_8000_default_lon,
        func_unit_cord=_unit_lon_8000,
        func_multiplier=lon_multiplier_lv)

    mesh_5000_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_5000_default_lat,
        func_unit_cord=_unit_lat_5000,
        func_multiplier=lat_multiplier_lv)

    mesh_5000_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_5000_default_lon,
        func_unit_cord=_unit_lon_5000,
        func_multiplier=lon_multiplier_lv)

    mesh_4000_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_4000_default_lat,
        func_unit_cord=_unit_lat_4000,
        func_multiplier=lat_multiplier_lv)

    mesh_4000_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_4000_default_lon,
        func_unit_cord=_unit_lon_4000,
        func_multiplier=lon_multiplier_lv)

    mesh_2500_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_2500_default_lat,
        func_unit_cord=_unit_lat_2500,
        func_multiplier=lat_multiplier_lv)

    mesh_2500_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_2500_default_lon,
        func_unit_cord=_unit_lon_2500,
        func_multiplier=lon_multiplier_lv)

    mesh_2000_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_2000_default_lat,
        func_unit_cord=_unit_lat_2000,
        func_multiplier=lat_multiplier_lv)

    mesh_2000_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_2000_default_lon,
        func_unit_cord=_unit_lon_2000,
        func_multiplier=lon_multiplier_lv)

    mesh_lv3_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv3_default_lat,
        func_unit_cord=_unit_lat_lv3,
        func_multiplier=lat_multiplier_lv)

    mesh_lv3_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv3_default_lon,
        func_unit_cord=_unit_lon_lv3,
        func_multiplier=lon_multiplier_lv)

    mesh_lv4_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv4_default_lat,
        func_unit_cord=_unit_lat_lv4,
        func_multiplier=lat_multiplier_lv)

    mesh_lv4_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv4_default_lon,
        func_unit_cord=_unit_lon_lv4,
        func_multiplier=lon_multiplier_lv)

    mesh_lv5_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv5_default_lat,
        func_unit_cord=_unit_lat_lv5,
        func_multiplier=lat_multiplier_lv)

    mesh_lv5_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv5_default_lon,
        func_unit_cord=_unit_lon_lv5,
        func_multiplier=lon_multiplier_lv)

    mesh_lv6_lat = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv6_default_lat,
        func_unit_cord=_unit_lat_lv6,
        func_multiplier=lat_multiplier_lv)

    mesh_lv6_lon = _functools.partial(
        mesh_cord,
        func_higher_cord=mesh_lv6_default_lon,
        func_unit_cord=_unit_lon_lv6,
        func_multiplier=lon_multiplier_lv)

    level = to_meshlevel(meshcode)

    if level == 1:
        return mesh_lv1_lat(), mesh_lv1_lon()

    if level == 40000:
        return mesh_40000_lat(), mesh_40000_lon()

    if level == 20000:
        return mesh_20000_lat(), mesh_20000_lon()

    if level == 16000:
        return mesh_16000_lat(), mesh_16000_lon()

    if level == 2:
        return mesh_lv2_lat(), mesh_lv2_lon()

    if level == 8000:
        return mesh_8000_lat(), mesh_8000_lon()

    if level == 5000:
        return mesh_5000_lat(), mesh_5000_lon()

    if level == 4000:
        return mesh_4000_lat(), mesh_4000_lon()

    if level == 2500:
        return mesh_2500_lat(), mesh_2500_lon()

    if level == 2000:
        return mesh_2000_lat(), mesh_2000_lon()

    if level == 3:
        return mesh_lv3_lat(), mesh_lv3_lon()

    if level == 4:
        return mesh_lv4_lat(), mesh_lv4_lon()

    if level == 5:
        return mesh_lv5_lat(), mesh_lv5_lon()

    if level == 6:
        return mesh_lv6_lat(), mesh_lv6_lon()

    raise ValueError("the level is unsupported.")

Response: Convert a meshcode to a meshpoint.


Instruction:
Can you implement a function in Python 3 that
checks the permissions http method and login state.

Response:
def check(func):
    """
    Check the permissions, http method and login state.
    """
    def iCheck(request, *args, **kwargs):
        if not request.method == "POST":
            return HttpResponseBadRequest("Must be POST request.")
        follow = func(request, *args, **kwargs)
        if request.is_ajax():
            return HttpResponse('ok')
        try:
            if 'next' in request.GET:
                return HttpResponseRedirect(request.GET.get('next'))
            if 'next' in request.POST:
                return HttpResponseRedirect(request.POST.get('next'))
            return HttpResponseRedirect(follow.target.get_absolute_url())
        except (AttributeError, TypeError):
            if 'HTTP_REFERER' in request.META:
                return HttpResponseRedirect(request.META.get('HTTP_REFERER', '/'))
            if follow:
                return HttpResponseServerError('"%s" object of type ``%s`` has no method ``get_absolute_url()``.' % (
                    unicode(follow.target), follow.target.__class__))
            return HttpResponseServerError('No follow object and `next` parameter found.')
    return iCheck


Instruction:
Write a Python 3 function for
registering a model class to be follow - able.

Response:
def register(model, field_name=None, related_name=None, lookup_method_name='get_follows'):
    """
    This registers any model class to be follow-able.

    """
    if model in registry:
        return

    registry.append(model)

    if not field_name:
        field_name = 'target_%s' % model._meta.module_name

    if not related_name:
        related_name = 'follow_%s' % model._meta.module_name

    field = ForeignKey(model, related_name=related_name, null=True,
        blank=True, db_index=True)

    field.contribute_to_class(Follow, field_name)
    setattr(model, lookup_method_name, get_followers_for_object)
    model_map[model] = [related_name, field_name]


Instruction:
Make a summary of the following Python 3 code
def follow(user, obj):
    follow, created = Follow.objects.get_or_create(user, obj)
    return follow

Response: Make a user follow an object


Instruction:
Write a Python 3 function for
making a user unfollow an object

Response:
def unfollow(user, obj):
    """ Make a user unfollow an object """
    try:
        follow = Follow.objects.get_follows(obj).get(user=user)
        follow.delete()
        return follow
    except Follow.DoesNotExist:
        pass


Instruction:
Explain what the following Python 3 code does
def validate_q_time(self, value):
        if value:
            try:
                range = utils.parse_datetime_range_to_solr(value)
                return range
            except Exception as e:
                raise serializers.ValidationError(e.message)

        return value

Response: Validate the q_time field.


Instruction:
Write a Python 3 script to
validate the value of the QGE field.

Response:
def validate_q_geo(self, value):
        """
        Would be for example: [-90,-180 TO 90,180]
        """
        if value:
            try:
                rectangle = utils.parse_geo_box(value)
                return "[{0},{1} TO {2},{3}]".format(
                    rectangle.bounds[0],
                    rectangle.bounds[1],
                    rectangle.bounds[2],
                    rectangle.bounds[3],
                )
            except Exception as e:
                raise serializers.ValidationError(e.message)

        return value


Instruction:
Implement a function in Python 3 to
validate a time filter.

Response:
def validate_a_time_filter(self, value):
        """
        Would be for example: [2013-03-01 TO 2013-04-01:00:00:00] and/or [* TO *]
        """
        if value:
            try:
                utils.parse_datetime_range(value)
            except Exception as e:
                raise serializers.ValidationError(e.message)

        return value


Instruction:
Write a Python 3 function for
returning the name of the field on the model.

Response:
def fname(self, model_or_obj_or_qs):
        """
        Return the field name on the :class:`Follow` model for ``model_or_obj_or_qs``.
        """
        if isinstance(model_or_obj_or_qs, QuerySet):
            _, fname = model_map[model_or_obj_or_qs.model]
        else:
            cls = model_or_obj_or_qs if inspect.isclass(model_or_obj_or_qs) else model_or_obj_or_qs.__class__
            _, fname = model_map[cls]
        return fname


Instruction:
Can you generate the documentation for the following Python 3 function
def create(self, user, obj, **kwargs):
        follow = Follow(user=user)
        follow.target = obj
        follow.save()
        return follow

Response: Create a new follow link between a user and an object.


Instruction:
Here you have a function in Python 3, explain what it does
def get_or_create(self, user, obj, **kwargs):
        if not self.is_following(user, obj):
            return self.create(user, obj, **kwargs), True
        return self.get_follows(obj).get(user=user), False

Response: Returns a tuple with the Follow and whether the user is following the object.


Instruction:
Given the following Python 3 function, write the documentation
def is_following(self, user, obj):
        if isinstance(user, AnonymousUser):
            return False
        return 0 < self.get_follows(obj).filter(user=user).count()

Response: Returns True if user is following obj


Instruction:
How would you code a function in Python 3 to
return all the followers of a model or a queryset.

Response:
def get_follows(self, model_or_obj_or_qs):
        """
        Returns all the followers of a model, an object or a queryset.
        """
        fname = self.fname(model_or_obj_or_qs)

        if isinstance(model_or_obj_or_qs, QuerySet):
            return self.filter(**{'%s__in' % fname: model_or_obj_or_qs})

        if inspect.isclass(model_or_obj_or_qs):
            return self.exclude(**{fname:None})

        return self.filter(**{fname:model_or_obj_or_qs})


Instruction:
Create a Python 3 function to
create the event regressors for one event type.

Response:
def create_event_regressors(self, event_times_indices, covariates = None, durations = None):
        """create_event_regressors creates the part of the design matrix corresponding to one event type.

            :param event_times_indices: indices in the resampled data, on which the events occurred.
            :type event_times_indices: numpy array, (nr_events)
            :param covariates: covariates belonging to this event type. If None, covariates with a value of 1 for all events are created and used internally.
            :type covariates: numpy array, (nr_events)
            :param durations: durations belonging to this event type. If None, durations with a value of 1 sample for all events are created and used internally.
            :type durations: numpy array, (nr_events)
            :returns: This event type's part of the design matrix.
        """

        # check covariates
        if covariates is None:
            covariates = np.ones(self.event_times_indices.shape)

        # check/create durations, convert from seconds to samples time, and compute mean duration for this event type.
        if durations is None:
            durations = np.ones(self.event_times_indices.shape)
        else:
            durations = np.round(durations*self.deconvolution_frequency).astype(int)
        mean_duration = np.mean(durations)

        # set up output array
        regressors_for_event = np.zeros((self.deconvolution_interval_size, self.resampled_signal_size))

        # fill up output array by looping over events.
        for cov, eti, dur in zip(covariates, event_times_indices, durations):
            valid = True
            if eti < 0:
                self.logger.debug('deconv samples are starting before the data starts.')
                valid = False
            if eti+self.deconvolution_interval_size > self.resampled_signal_size:
                self.logger.debug('deconv samples are continuing after the data stops.')
                valid = False
            if eti > self.resampled_signal_size:
                self.logger.debug('event falls outside of the scope of the data.')
                valid = False

            if valid: # only incorporate sensible events.
                # calculate the design matrix that belongs to this event.
                this_event_design_matrix = (np.diag(np.ones(self.deconvolution_interval_size)) * cov)
                over_durations_dm = np.copy(this_event_design_matrix)
                if dur > 1: # if this event has a non-unity duration, duplicate the stick regressors in the time direction
                    for d in np.arange(1,dur):
                        over_durations_dm[d:] += this_event_design_matrix[:-d]
                    # and correct for differences in durations between different regressor types.
                    over_durations_dm /= mean_duration
                # add the designmatrix for this event to the full design matrix for this type of event.
                regressors_for_event[:,eti:int(eti+self.deconvolution_interval_size)] += over_durations_dm

        return regressors_for_event


Instruction:
Write a Python 3 script for
creating the design matrix for each covariate.

Response:
def create_design_matrix(self, demean = False, intercept = True):
        """create_design_matrix calls create_event_regressors for each of the covariates in the self.covariates dict. self.designmatrix is created and is shaped (nr_regressors, self.resampled_signal.shape[-1])
        """
        self.design_matrix = np.zeros((int(self.number_of_event_types*self.deconvolution_interval_size), self.resampled_signal_size))

        for i, covariate in enumerate(self.covariates.keys()):
            # document the creation of the designmatrix step by step
            self.logger.debug('creating regressor for ' + covariate)
            indices = np.arange(i*self.deconvolution_interval_size,(i+1)*self.deconvolution_interval_size, dtype = int)
            # here, we implement the dot-separated encoding of events and covariates
            if len(covariate.split('.')) > 0:
                which_event_time_indices = covariate.split('.')[0]
            else:
                which_event_time_indices = covariate
            self.design_matrix[indices] = self.create_event_regressors( self.event_times_indices[which_event_time_indices],
                                                                        self.covariates[covariate],
                                                                        self.durations[which_event_time_indices])

        if demean:
            # we expect the data to be demeaned.
            # it's an option whether the regressors should be, too
            self.design_matrix = (self.design_matrix.T - self.design_matrix.mean(axis = -1)).T
        if intercept:
            # similarly, intercept is a choice.
            self.design_matrix = np.vstack((self.design_matrix, np.ones((1,self.design_matrix.shape[-1]))))

        self.logger.debug('created %s design_matrix' % (str(self.design_matrix.shape)))


Instruction:
Explain what the following Python 3 code does
def add_continuous_regressors_to_design_matrix(self, regressors):
        previous_design_matrix_shape = self.design_matrix.shape
        if len(regressors.shape) == 1:
            regressors = regressors[np.newaxis, :]
        if regressors.shape[1] != self.resampled_signal.shape[1]:
            self.logger.warning('additional regressor shape %s does not conform to designmatrix shape %s' % (regressors.shape, self.resampled_signal.shape))
        # and, an vstack append
        self.design_matrix = np.vstack((self.design_matrix, regressors))
        self.logger.debug('added %s continuous regressors to %s design_matrix, shape now %s' % (str(regressors.shape), str(previous_design_matrix_shape), str(self.design_matrix.shape)))

Response: Adds continuously sampled regressors to the design matrix.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def regress(self, method = 'lstsq'):

        if method is 'lstsq':
            self.betas, residuals_sum, rank, s = LA.lstsq(self.design_matrix.T, self.resampled_signal.T)
            self.residuals = self.resampled_signal - self.predict_from_design_matrix(self.design_matrix)
        elif method is 'sm_ols':
            import statsmodels.api as sm

            assert self.resampled_signal.shape[0] == 1, \
                    'signal input into statsmodels OLS cannot contain multiple signals at once, present shape %s' % str(self.resampled_signal.shape)
            model = sm.OLS(np.squeeze(self.resampled_signal),self.design_matrix.T)
            results = model.fit()
            # make betas and residuals that are compatible with the LA.lstsq type.
            self.betas = np.array(results.params).reshape((self.design_matrix.shape[0], self.resampled_signal.shape[0]))
            self.residuals = np.array(results.resid).reshape(self.resampled_signal.shape)

        self.logger.debug('performed %s regression on %s design_matrix and %s signal' % (method, str(self.design_matrix.shape), str(self.resampled_signal.shape)))

Response: regress computes the linear least squares regression of the design matrix on the data.


Instruction:
Implement a Python 3 function for
performing k - folds ridge regression on the design_matrix.

Response:
def ridge_regress(self, cv = 20, alphas = None ):
        """perform k-folds cross-validated ridge regression on the design_matrix. To be used when the design matrix contains very collinear regressors. For cross-validation and ridge fitting, we use sklearn's RidgeCV functionality. Note: intercept is not fit, and data are not prenormalized.

            :param cv: cross-validated folds, inherits RidgeCV cv argument's functionality.
            :type cv: int, standard = 20
            :param alphas: values of penalization parameter to be traversed by the procedure, inherits RidgeCV cv argument's functionality. Standard value, when parameter is None, is np.logspace(7, 0, 20)
            :type alphas: numpy array, from >0 to 1.
            :returns: instance variables 'betas' (nr_betas x nr_signals) and 'residuals' (nr_signals x nr_samples) are created.
        """
        if alphas is None:
            alphas = np.logspace(7, 0, 20)
        self.rcv = linear_model.RidgeCV(alphas=alphas,
                fit_intercept=False,
                cv=cv)
        self.rcv.fit(self.design_matrix.T, self.resampled_signal.T)

        self.betas = self.rcv.coef_.T
        self.residuals = self.resampled_signal - self.rcv.predict(self.design_matrix.T)

        self.logger.debug('performed ridge regression on %s design_matrix and %s signal, resulting alpha value is %f' % (str(self.design_matrix.shape), str(self.resampled_signal.shape), self.rcv.alpha_))


Instruction:
How would you explain what the following Python 3 function does
def betas_for_cov(self, covariate = '0'):
        # find the index in the designmatrix of the current covariate
        this_covariate_index = list(self.covariates.keys()).index(covariate)
        return self.betas[int(this_covariate_index*self.deconvolution_interval_size):int((this_covariate_index+1)*self.deconvolution_interval_size)]

Response: returns the beta values associated with a specific covariate


Instruction:
Can you generate the documentation for the following Python 3 function
def betas_for_events(self):
        self.betas_per_event_type = np.zeros((len(self.covariates), self.deconvolution_interval_size, self.resampled_signal.shape[0]))
        for i, covariate in enumerate(self.covariates.keys()):
            self.betas_per_event_type[i] = self.betas_for_cov(covariate)

Response: Creates an array of betas per event type for each outcome.


Instruction:
Given the following Python 3 function, write the documentation
def predict_from_design_matrix(self, design_matrix):
        # check if we have already run the regression - which is necessary
        assert hasattr(self, 'betas'), 'no betas found, please run regression before prediction'
        assert design_matrix.shape[0] == self.betas.shape[0], \
                    'designmatrix needs to have the same number of regressors as the betas already calculated'

        # betas = np.copy(self.betas.T, order="F", dtype = np.float32)
        # f_design_matrix = np.copy(design_matrix, order = "F", dtype = np.float32)

        prediction = np.dot(self.betas.astype(np.float32).T, design_matrix.astype(np.float32))

        return prediction

Response: predict_from_design_matrix predicts signals given a design matrix.


Instruction:
Here you have a function in Python 3, explain what it does
def calculate_rsq(self):
        assert hasattr(self, 'betas'), 'no betas found, please run regression before rsq'

        explained_times = self.design_matrix.sum(axis = 0) != 0

        explained_signal = self.predict_from_design_matrix(self.design_matrix)
        self.rsq = 1.0 - np.sum((explained_signal[:,explained_times] - self.resampled_signal[:,explained_times])**2, axis = -1) / np.sum(self.resampled_signal[:,explained_times].squeeze()**2, axis = -1)
        self.ssr = np.sum((explained_signal[:,explained_times] - self.resampled_signal[:,explained_times])**2, axis = -1)
        return np.squeeze(self.rsq)

Response: calculate_rsq calculates coefficient of determination or r - squared defined here as 1. 0 - SS_res / SS_tot. rsq is only calculated for those timepoints in the data for which the design matrix is non - zero. ssr is only calculated for those timepoints in the data for which the design matrix is non - zero.


Instruction:
Given the following Python 3 function, write the documentation
def bootstrap_on_residuals(self, nr_repetitions = 1000):
        assert self.resampled_signal.shape[0] == 1, \
                    'signal input into bootstrap_on_residuals cannot contain signals from multiple channels at once, present shape %s' % str(self.resampled_signal.shape)
        assert hasattr(self, 'betas'), 'no betas found, please run regression before bootstrapping'

        # create bootstrap data by taking the residuals
        bootstrap_data = np.zeros((self.resampled_signal_size, nr_repetitions))
        explained_signal = self.predict_from_design_matrix(self.design_matrix).T

        for x in range(bootstrap_data.shape[-1]): # loop over bootstrapsamples
            bootstrap_data[:,x] = (self.residuals.T[np.random.permutation(self.resampled_signal_size)] + explained_signal).squeeze()

        self.bootstrap_betas, bs_residuals, rank, s = LA.lstsq(self.design_matrix.T, bootstrap_data)

        self.bootstrap_betas_per_event_type = np.zeros((len(self.covariates), self.deconvolution_interval_size, nr_repetitions))

        for i, covariate in enumerate(list(self.covariates.keys())):
            # find the index in the designmatrix of the current covariate
            this_covariate_index = list(self.covariates.keys()).index(covariate)
            self.bootstrap_betas_per_event_type[i] = self.bootstrap_betas[this_covariate_index*self.deconvolution_interval_size:(this_covariate_index+1)*self.deconvolution_interval_size]

Response: This function computes the bootstraps on the residuals of the current betas and creates the residuals for each event.


Instruction:
How would you implement a function in Python 3 that
indexes all layers in the Django cache.

Response:
def index_cached_layers(self):
    """
    Index and unindex all layers in the Django cache (Index all layers who have been checked).
    """
    from hypermap.aggregator.models import Layer

    if SEARCH_TYPE == 'solr':
        from hypermap.aggregator.solr import SolrHypermap
        solrobject = SolrHypermap()
    else:
        from hypermap.aggregator.elasticsearch_client import ESHypermap
        from elasticsearch import helpers
        es_client = ESHypermap()

    layers_cache = cache.get('layers')
    deleted_layers_cache = cache.get('deleted_layers')

    # 1. added layers cache
    if layers_cache:
        layers_list = list(layers_cache)
        LOGGER.debug('There are %s layers in cache: %s' % (len(layers_list), layers_list))

        batch_size = settings.REGISTRY_SEARCH_BATCH_SIZE
        batch_lists = [layers_list[i:i+batch_size] for i in range(0, len(layers_list), batch_size)]

        for batch_list_ids in batch_lists:
            layers = Layer.objects.filter(id__in=batch_list_ids)

            if batch_size > len(layers):
                batch_size = len(layers)

            LOGGER.debug('Syncing %s/%s layers to %s: %s' % (batch_size, len(layers_cache), layers, SEARCH_TYPE))

            try:
                # SOLR
                if SEARCH_TYPE == 'solr':
                    success, layers_errors_ids = solrobject.layers_to_solr(layers)
                    if success:
                        # remove layers from cache here
                        layers_cache = layers_cache.difference(set(batch_list_ids))
                        LOGGER.debug('Removing layers with id %s from cache' % batch_list_ids)
                        cache.set('layers', layers_cache)
                # ES
                elif SEARCH_TYPE == 'elasticsearch':
                    with_bulk, success = True, False
                    layers_to_index = [es_client.layer_to_es(layer, with_bulk) for layer in layers]
                    message = helpers.bulk(es_client.es, layers_to_index)

                    # Check that all layers where indexed...if not, don't clear cache.
                    # TODO: Check why es does not index all layers at first.
                    len_indexed_layers = message[0]
                    if len_indexed_layers == len(layers):
                        LOGGER.debug('%d layers indexed successfully' % (len_indexed_layers))
                        success = True
                    if success:
                        # remove layers from cache here
                        layers_cache = layers_cache.difference(set(batch_list_ids))
                        cache.set('layers', layers_cache)
                else:
                    raise Exception("Incorrect SEARCH_TYPE=%s" % SEARCH_TYPE)
            except Exception as e:
                LOGGER.error('Layers were NOT indexed correctly')
                LOGGER.error(e, exc_info=True)
    else:
        LOGGER.debug('No cached layers to add in search engine.')

    # 2. deleted layers cache
    if deleted_layers_cache:
        layers_list = list(deleted_layers_cache)
        LOGGER.debug('There are %s layers in cache for deleting: %s' % (len(layers_list), layers_list))
        # TODO implement me: batch layer index deletion
        for layer_id in layers_list:
            # SOLR
            if SEARCH_TYPE == 'solr':
                if Layer.objects.filter(pk=layer_id).exists():
                    layer = Layer.objects.get(id=layer_id)
                    unindex_layer(layer.id, use_cache=False)
                    deleted_layers_cache = deleted_layers_cache.difference(set([layer_id]))
                    cache.set('deleted_layers', deleted_layers_cache)
            else:
                # TODO implement me
                raise NotImplementedError
    else:
        LOGGER.debug('No cached layers to remove in search engine.')


Instruction:
How would you code a function in Python 3 to
remove all checks from a service.

Response:
def remove_service_checks(self, service_id):
    """
    Remove all checks from a service.
    """
    from hypermap.aggregator.models import Service
    service = Service.objects.get(id=service_id)

    service.check_set.all().delete()
    layer_to_process = service.layer_set.all()
    for layer in layer_to_process:
        layer.check_set.all().delete()


Instruction:
Here you have a function in Python 3, explain what it does
def index_service(self, service_id):

    from hypermap.aggregator.models import Service
    service = Service.objects.get(id=service_id)

    if not service.is_valid:
        LOGGER.debug('Not indexing service with id %s in search engine as it is not valid' % service.id)
        return

    LOGGER.debug('Indexing service %s' % service.id)
    layer_to_process = service.layer_set.all()

    for layer in layer_to_process:
        if not settings.REGISTRY_SKIP_CELERY:
            index_layer(layer.id, use_cache=True)
        else:
            index_layer(layer.id)

Response: Index a service in search engine.


Instruction:
Here you have a function in Python 3, explain what it does
def index_layer(self, layer_id, use_cache=False):

    from hypermap.aggregator.models import Layer
    layer = Layer.objects.get(id=layer_id)

    if not layer.is_valid:
        LOGGER.debug('Not indexing or removing layer with id %s in search engine as it is not valid' % layer.id)
        unindex_layer(layer.id, use_cache)
        return

    if layer.was_deleted:
        LOGGER.debug('Not indexing or removing layer with id %s in search engine as was_deleted is true' % layer.id)
        unindex_layer(layer.id, use_cache)
        return

    # 1. if we use cache
    if use_cache:
        LOGGER.debug('Caching layer with id %s for syncing with search engine' % layer.id)
        layers = cache.get('layers')
        if layers is None:
            layers = set([layer.id])
        else:
            layers.add(layer.id)
        cache.set('layers', layers)
        return

    # 2. if we don't use cache
    # TODO: Make this function more DRY
    # by abstracting the common bits.
    if SEARCH_TYPE == 'solr':
        from hypermap.aggregator.solr import SolrHypermap
        LOGGER.debug('Syncing layer %s to solr' % layer.name)
        solrobject = SolrHypermap()
        success, message = solrobject.layer_to_solr(layer)
        # update the error message if using celery
        if not settings.REGISTRY_SKIP_CELERY:
            if not success:
                self.update_state(
                    state=states.FAILURE,
                    meta=message
                    )
                raise Ignore()
    elif SEARCH_TYPE == 'elasticsearch':
        from hypermap.aggregator.elasticsearch_client import ESHypermap
        LOGGER.debug('Syncing layer %s to es' % layer.name)
        esobject = ESHypermap()
        success, message = esobject.layer_to_es(layer)
        # update the error message if using celery
        if not settings.REGISTRY_SKIP_CELERY:
            if not success:
                self.update_state(
                    state=states.FAILURE,
                    meta=message
                    )
                raise Ignore()

Response: Index a layer in the search backend.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def unindex_layers_with_issues(self, use_cache=False):
    from hypermap.aggregator.models import Issue, Layer, Service
    from django.contrib.contenttypes.models import ContentType

    layer_type = ContentType.objects.get_for_model(Layer)
    service_type = ContentType.objects.get_for_model(Service)

    for issue in Issue.objects.filter(content_type__pk=layer_type.id):
        unindex_layer(issue.content_object.id, use_cache)

    for issue in Issue.objects.filter(content_type__pk=service_type.id):
        for layer in issue.content_object.layer_set.all():
            unindex_layer(layer.id, use_cache)

Response: Remove the index for layers in search backend which are linked to an issue.


Instruction:
How would you implement a function in Python 3 that
removes the index for a layer in the search backend.

Response:
def unindex_layer(self, layer_id, use_cache=False):
    """
    Remove the index for a layer in the search backend.
    If cache is set, append it to the list of removed layers, if it isn't send the transaction right away.
    """

    from hypermap.aggregator.models import Layer
    layer = Layer.objects.get(id=layer_id)

    if use_cache:
        LOGGER.debug('Caching layer with id %s for being removed from search engine' % layer.id)
        deleted_layers = cache.get('deleted_layers')
        if deleted_layers is None:
            deleted_layers = set([layer.id])
        else:
            deleted_layers.add(layer.id)
        cache.set('deleted_layers', deleted_layers)
        return

    if SEARCH_TYPE == 'solr':
        from hypermap.aggregator.solr import SolrHypermap
        LOGGER.debug('Removing layer %s from solr' % layer.id)
        try:
            solrobject = SolrHypermap()
            solrobject.remove_layer(layer.uuid)
        except Exception:
            LOGGER.error('Layer NOT correctly removed from Solr')
    elif SEARCH_TYPE == 'elasticsearch':
        # TODO implement me
        pass


Instruction:
Create a Python 3 function to
index all layers in search engine.

Response:
def index_all_layers(self):
    """
    Index all layers in search engine.
    """
    from hypermap.aggregator.models import Layer

    if not settings.REGISTRY_SKIP_CELERY:
        layers_cache = set(Layer.objects.filter(is_valid=True).values_list('id', flat=True))
        deleted_layers_cache = set(Layer.objects.filter(is_valid=False).values_list('id', flat=True))
        cache.set('layers', layers_cache)
        cache.set('deleted_layers', deleted_layers_cache)
    else:
        for layer in Layer.objects.all():
            index_layer(layer.id)


Instruction:
Create a Python 3 function to
update the last num_layers layers in the WorldMap service.

Response:
def update_last_wm_layers(self, service_id, num_layers=10):
    """
    Update and index the last added and deleted layers (num_layers) in WorldMap service.
    """
    from hypermap.aggregator.models import Service

    LOGGER.debug(
        'Updating the index the last %s added and %s deleted layers in WorldMap service'
        % (num_layers, num_layers)
                )
    service = Service.objects.get(id=service_id)
    # TODO raise error if service type is not WM type
    if service.type == 'Hypermap:WorldMapLegacy':
        from hypermap.aggregator.models import update_layers_wm_legacy as update_layers_wm
    if service.type == 'Hypermap:WorldMap':
        from hypermap.aggregator.models import update_layers_geonode_wm as update_layers_wm

    update_layers_wm(service, num_layers)

    # Remove in search engine last num_layers that were deleted
    LOGGER.debug('Removing the index for the last %s deleted layers' % num_layers)
    layer_to_unindex = service.layer_set.filter(was_deleted=True).order_by('-last_updated')[0:num_layers]
    for layer in layer_to_unindex:
        if not settings.REGISTRY_SKIP_CELERY:
            unindex_layer(layer.id, use_cache=True)
        else:
            unindex_layer(layer.id)

    # Add/Update in search engine last num_layers that were added
    LOGGER.debug('Adding/Updating the index for the last %s added layers' % num_layers)
    layer_to_index = service.layer_set.filter(was_deleted=False).order_by('-last_updated')[0:num_layers]
    for layer in layer_to_index:
        if not settings.REGISTRY_SKIP_CELERY:
            index_layer(layer.id, use_cache=True)
        else:
            index_layer(layer.id)


Instruction:
Write a Python 3 function for
returning a WKT Polygon of a simple bounding box list

Response:
def bbox2wktpolygon(bbox):
    """
    Return OGC WKT Polygon of a simple bbox list
    """

    try:
        minx = float(bbox[0])
        miny = float(bbox[1])
        maxx = float(bbox[2])
        maxy = float(bbox[3])

    except:
        LOGGER.debug("Invalid bbox, setting it to a zero POLYGON")
        minx = 0
        miny = 0
        maxx = 0
        maxy = 0

    return 'POLYGON((%.2f %.2f, %.2f %.2f, %.2f %.2f, %.2f %.2f, %.2f %.2f))' \
        % (minx, miny, minx, maxy, maxx, maxy, maxx, miny, minx, miny)


Instruction:
Write a Python 3 function for
creating a csw : Record XML document from harvested metadata

Response:
def create_metadata_record(**kwargs):
    """
    Create a csw:Record XML document from harvested metadata
    """

    if 'srs' in kwargs:
        srs = kwargs['srs']
    else:
        srs = '4326'

    modified = '%sZ' % datetime.datetime.utcnow().isoformat().split('.')[0]

    nsmap = Namespaces().get_namespaces(['csw', 'dc', 'dct', 'ows'])

    e = etree.Element(nspath_eval('csw:Record', nsmap), nsmap=nsmap)

    etree.SubElement(e, nspath_eval('dc:identifier', nsmap)).text = kwargs['identifier']
    etree.SubElement(e, nspath_eval('dc:title', nsmap)).text = kwargs['title']
    if 'alternative' in kwargs:
        etree.SubElement(e, nspath_eval('dct:alternative', nsmap)).text = kwargs['alternative']
    etree.SubElement(e, nspath_eval('dct:modified', nsmap)).text = modified
    etree.SubElement(e, nspath_eval('dct:abstract', nsmap)).text = kwargs['abstract']
    etree.SubElement(e, nspath_eval('dc:type', nsmap)).text = kwargs['type']
    etree.SubElement(e, nspath_eval('dc:format', nsmap)).text = kwargs['format']
    etree.SubElement(e, nspath_eval('dc:source', nsmap)).text = kwargs['source']

    if 'relation' in kwargs:
        etree.SubElement(e, nspath_eval('dc:relation', nsmap)).text = kwargs['relation']

    if 'keywords' in kwargs:
        if kwargs['keywords'] is not None:
            for keyword in kwargs['keywords']:
                etree.SubElement(e, nspath_eval('dc:subject', nsmap)).text = keyword

    for link in kwargs['links']:
        etree.SubElement(e, nspath_eval('dct:references', nsmap), scheme=link[0]).text = link[1]

    bbox2 = loads(kwargs['wkt_geometry']).bounds
    bbox = etree.SubElement(e, nspath_eval('ows:BoundingBox', nsmap),
                            crs='http://www.opengis.net/def/crs/EPSG/0/%s' % srs,
                            dimensions='2')

    etree.SubElement(bbox, nspath_eval('ows:LowerCorner', nsmap)).text = '%s %s' % (bbox2[1], bbox2[0])
    etree.SubElement(bbox, nspath_eval('ows:UpperCorner', nsmap)).text = '%s %s' % (bbox2[3], bbox2[2])

    return etree.tostring(e, pretty_print=True)


Instruction:
Can you generate the documentation for the following Python 3 function
def gen_anytext(*args):

    bag = []

    for term in args:
        if term is not None:
            if isinstance(term, list):
                for term2 in term:
                    if term2 is not None:
                        bag.append(term2)
            else:
                bag.append(term)
    return ' '.join(bag)

Response: Convenience function to create bag of words for anytext property


Instruction:
Here you have a function in Python 3, explain what it does
def update_layers_wmts(service):
    try:
        wmts = WebMapTileService(service.url)

        # set srs
        # WMTS is always in 4326
        srs, created = SpatialReferenceSystem.objects.get_or_create(code='EPSG:4326')
        service.srs.add(srs)

        service.update_validity()

        layer_names = list(wmts.contents)
        layer_n = 0
        total = len(layer_names)
        for layer_name in layer_names:
            ows_layer = wmts.contents[layer_name]
            LOGGER.debug('Updating layer %s' % ows_layer.name)
            layer, created = Layer.objects.get_or_create(name=ows_layer.name, service=service, catalog=service.catalog)
            if layer.active:
                links = [['OGC:WMTS', service.url],
                         ['OGC:WMTS', settings.SITE_URL.rstrip('/') + '/' + layer.get_url_endpoint()]]
                layer.type = 'OGC:WMTS'
                layer.title = ows_layer.title
                layer.abstract = ows_layer.abstract
                # keywords
                # @tomkralidis wmts does not seem to support this attribute
                keywords = None
                if hasattr(ows_layer, 'keywords'):
                    keywords = ows_layer.keywords
                    for keyword in keywords:
                        layer.keywords.add(keyword)
                layer.url = service.url
                layer.page_url = layer.get_absolute_url
                links.append([
                    'WWW:LINK',
                    settings.SITE_URL.rstrip('/') + layer.page_url
                ])
                bbox = list(ows_layer.boundingBoxWGS84 or (-179.0, -89.0, 179.0, 89.0))
                layer.bbox_x0 = bbox[0]
                layer.bbox_y0 = bbox[1]
                layer.bbox_x1 = bbox[2]
                layer.bbox_y1 = bbox[3]
                layer.wkt_geometry = bbox2wktpolygon(bbox)
                layer.xml = create_metadata_record(
                    identifier=str(layer.uuid),
                    source=service.url,
                    links=links,
                    format='OGC:WMS',
                    type=layer.csw_type,
                    relation=service.id_string,
                    title=ows_layer.title,
                    alternative=ows_layer.name,
                    abstract=layer.abstract,
                    keywords=keywords,
                    wkt_geometry=layer.wkt_geometry
                )
                layer.anytext = gen_anytext(layer.title, layer.abstract, keywords)
                layer.save()
                # dates
                add_mined_dates(layer)
            layer_n = layer_n + 1
            # exits if DEBUG_SERVICES
            LOGGER.debug("Updating layer n. %s/%s" % (layer_n, total))
            if DEBUG_SERVICES and layer_n == DEBUG_LAYER_NUMBER:
                return
    except Exception as err:
        message = "update_layers_wmts: {0}".format(
            err
        )
        check = Check(
            content_object=service,
            success=False,
            response_time=0,
            message=message
        )
        check.save()

Response: Update layers for an OGC WMTS service.


Instruction:
Implement a Python 3 function for
updating layers for a WorldMap instance.

Response:
def update_layers_geonode_wm(service, num_layers=None):
    """
    Update layers for a WorldMap instance.
    Sample endpoint: http://localhost:8000/
    """
    wm_api_url = urlparse.urljoin(service.url, 'worldmap/api/2.8/layer/?format=json')

    if num_layers:
        total = num_layers
    else:
        response = requests.get(wm_api_url)
        data = json.loads(response.content)
        total = data['meta']['total_count']

    # set srs
    # WorldMap supports only 4326, 900913, 3857
    for crs_code in ['EPSG:4326', 'EPSG:900913', 'EPSG:3857']:
        srs, created = SpatialReferenceSystem.objects.get_or_create(code=crs_code)
        service.srs.add(srs)

    service.update_validity()

    layer_n = 0
    limit = 10

    for i in range(0, total, limit):
        try:
            url = (
                    '%s&order_by=-date&offset=%s&limit=%s' % (wm_api_url, i, limit)
            )
            LOGGER.debug('Fetching %s' % url)
            response = requests.get(url)
            data = json.loads(response.content)
            for row in data['objects']:
                typename = row['typename']
                # name = typename.split(':')[1]
                name = typename
                uuid = row['uuid']
                LOGGER.debug('Updating layer %s' % name)
                title = row['title']
                abstract = row['abstract']
                bbox = row['bbox']
                page_url = urlparse.urljoin(service.url, 'data/%s' % name)
                category = ''
                if 'topic_category' in row:
                    category = row['topic_category']
                username = ''
                if 'owner_username' in row:
                    username = row['owner_username']
                temporal_extent_start = ''
                if 'temporal_extent_start' in row:
                    temporal_extent_start = row['temporal_extent_start']
                temporal_extent_end = ''
                if 'temporal_extent_end' in row:
                    temporal_extent_end = row['temporal_extent_end']
                # we use the geoserver virtual layer getcapabilities for wm endpoint
                # TODO we should port make geoserver port configurable some way...
                # endpoint = urlparse.urljoin(service.url, 'geoserver/geonode/%s/wms?' % name)
                endpoint = urlparse.urljoin(service.url, 'geoserver/wms?')
                endpoint = endpoint.replace('8000', '8080')
                print endpoint
                if 'is_public' in row:
                    is_public = row['is_public']
                layer, created = Layer.objects.get_or_create(
                    service=service, catalog=service.catalog, name=name, uuid=uuid)
                if created:
                    LOGGER.debug('Added a new layer in registry: %s, %s' % (name, uuid))
                if layer.active:
                    links = [['Hypermap:WorldMap', endpoint]]
                    # update fields
                    layer.type = 'Hypermap:WorldMap'
                    layer.title = title
                    layer.abstract = abstract
                    layer.is_public = is_public
                    layer.url = endpoint
                    layer.page_url = page_url
                    # category and owner username
                    layer_wm, created = LayerWM.objects.get_or_create(layer=layer)
                    layer_wm.category = category
                    layer_wm.username = username
                    layer_wm.temporal_extent_start = temporal_extent_start
                    layer_wm.temporal_extent_end = temporal_extent_end
                    layer_wm.save()
                    # bbox [x0, y0, x1, y1]
                    # check if it is a valid bbox (TODO improve this check)
                    # bbox = bbox.replace('-inf', 'None')
                    # bbox = bbox.replace('inf', 'None')
                    # if bbox.count(',') == 3:
                    #    bbox_list = bbox[1:-1].split(',')
                    # else:
                    #    bbox_list = [None, None, None, None]
                    x0 = format_float(bbox[0])
                    x1 = format_float(bbox[1])
                    y0 = format_float(bbox[2])
                    y1 = format_float(bbox[3])
                    # In many cases for some reason to be fixed GeoServer has x coordinates flipped in WM.
                    x0, x1 = flip_coordinates(x0, x1)
                    y0, y1 = flip_coordinates(y0, y1)
                    layer.bbox_x0 = x0
                    layer.bbox_y0 = y0
                    layer.bbox_x1 = x1
                    layer.bbox_y1 = y1
                    # keywords
                    keywords = []
                    for keyword in row['keywords']:
                        keywords.append(keyword['name'])
                    layer.keywords.all().delete()
                    for keyword in keywords:
                        layer.keywords.add(keyword)
                    layer.wkt_geometry = bbox2wktpolygon([x0, y0, x1, y1])
                    layer.xml = create_metadata_record(
                        identifier=str(layer.uuid),
                        source=endpoint,
                        links=links,
                        format='Hypermap:WorldMap',
                        type=layer.csw_type,
                        relation=service.id_string,
                        title=layer.title,
                        alternative=name,
                        abstract=layer.abstract,
                        keywords=keywords,
                        wkt_geometry=layer.wkt_geometry
                    )
                    layer.anytext = gen_anytext(layer.title, layer.abstract, keywords)
                    layer.save()
                    # dates
                    add_mined_dates(layer)
                    add_metadata_dates_to_layer([layer_wm.temporal_extent_start, layer_wm.temporal_extent_end], layer)
                    layer_n = layer_n + 1
                    # exits if DEBUG_SERVICES
                    LOGGER.debug("Updated layer n. %s/%s" % (layer_n, total))
                    if DEBUG_SERVICES and layer_n == DEBUG_LAYER_NUMBER:
                        return

        except Exception as err:
            LOGGER.error('Error! %s' % err)

    # update deleted layers. For now we check the whole set of deleted layers
    # we should optimize it if the list will grow
    # TODO implement the actions application
    url = urlparse.urljoin(service.url, 'worldmap/api/2.8/actionlayerdelete/?format=json')
    LOGGER.debug('Fetching %s for detecting deleted layers' % url)
    try:
        response = requests.get(url)
        data = json.loads(response.content)
        for deleted_layer in data['objects']:
            if Layer.objects.filter(uuid=deleted_layer['args']).count() > 0:
                layer = Layer.objects.get(uuid=deleted_layer['args'])
                layer.was_deleted = True
                layer.save()
                LOGGER.debug('Layer %s marked as deleted' % layer.uuid)
    except Exception as err:
        LOGGER.error('Error! %s' % err)


Instruction:
Here you have a function in Python 3, explain what it does
def update_layers_warper(service):
    params = {'field': 'title', 'query': '', 'show_warped': '1', 'format': 'json'}
    headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
    request = requests.get(service.url, headers=headers, params=params)

    try:
        records = json.loads(request.content)
        total_pages = int(records['total_pages'])

        # set srs
        # Warper supports only 4326, 900913, 3857
        for crs_code in ['EPSG:4326', 'EPSG:900913', 'EPSG:3857']:
            srs, created = SpatialReferenceSystem.objects.get_or_create(code=crs_code)
            service.srs.add(srs)

        service.update_validity()

        for i in range(1, total_pages + 1):
            params = {'field': 'title', 'query': '', 'show_warped': '1', 'format': 'json', 'page': i}
            request = requests.get(service.url, headers=headers, params=params)
            records = json.loads(request.content)
            LOGGER.debug('Fetched %s' % request.url)
            layers = records['items']
            layer_n = 0
            total = len(layers)
            for layer in layers:
                name = layer['id']
                title = layer['title']
                abstract = layer['description']
                bbox = layer['bbox']
                # dates
                dates = []
                if 'published_date' in layer:
                    dates.append(layer['published_date'])
                if 'date_depicted' in layer:
                    dates.append(layer['date_depicted'])
                if 'depicts_year' in layer:
                    dates.append(layer['depicts_year'])
                if 'issue_year' in layer:
                    dates.append(layer['issue_year'])
                layer, created = Layer.objects.get_or_create(name=name, service=service, catalog=service.catalog)
                if layer.active:
                    # update fields
                    # links = [['OGC:WMTS', settings.SITE_URL.rstrip('/') + '/' + layer.get_url_endpoint()]]
                    layer.type = 'Hypermap:WARPER'
                    layer.title = title
                    layer.abstract = abstract
                    layer.is_public = True
                    layer.url = '%s/wms/%s?' % (service.url, name)
                    layer.page_url = '%s/%s' % (service.url, name)
                    # bbox
                    x0 = None
                    y0 = None
                    x1 = None
                    y1 = None
                    if bbox:
                        bbox_list = bbox.split(',')
                        x0 = format_float(bbox_list[0])
                        y0 = format_float(bbox_list[1])
                        x1 = format_float(bbox_list[2])
                        y1 = format_float(bbox_list[3])
                    layer.bbox_x0 = x0
                    layer.bbox_y0 = y0
                    layer.bbox_x1 = x1
                    layer.bbox_y1 = y1
                    layer.save()
                    # dates
                    add_mined_dates(layer)
                    add_metadata_dates_to_layer(dates, layer)
                layer_n = layer_n + 1
                # exits if DEBUG_SERVICES
                LOGGER.debug("Updating layer n. %s/%s" % (layer_n, total))
                if DEBUG_SERVICES and layer_n == DEBUG_LAYER_NUMBER:
                    return

    except Exception as err:
        message = "update_layers_warper: {0}. request={1} response={2}".format(
            err,
            service.url,
            request.text
        )
        check = Check(
            content_object=service,
            success=False,
            response_time=0,
            message=message
        )
        check.save()

Response: Update layers for a Warper service.


Instruction:
Create a Python 3 function for
updating the layers of an ESRI REST MapServer.

Response:
def update_layers_esri_mapserver(service, greedy_opt=False):
    """
    Update layers for an ESRI REST MapServer.
    Sample endpoint: https://gis.ngdc.noaa.gov/arcgis/rest/services/SampleWorldCities/MapServer/?f=json
    """
    try:
        esri_service = ArcMapService(service.url)
        # set srs
        # both mapserver and imageserver exposes just one srs at the service level
        # not sure if other ones are supported, for now we just store this one

        # not sure why this is needed, for now commenting out
        # if wkt_text:
        #     params = {'exact': 'True', 'error': 'True', 'mode': 'wkt', 'terms': wkt_text}
        #     req = requests.get('http://prj2epsg.org/search.json', params=params)
        #     object = json.loads(req.content)
        #     srs = int(object['codes'][0]['code'])

        srs_code = esri_service.spatialReference.wkid
        srs, created = SpatialReferenceSystem.objects.get_or_create(code=srs_code)
        service.srs.add(srs)

        service.update_validity()

        # check if it has a WMS interface
        if 'supportedExtensions' in esri_service._json_struct and greedy_opt:
            if 'WMSServer' in esri_service._json_struct['supportedExtensions']:
                # we need to change the url
                # http://cga1.cga.harvard.edu/arcgis/rest/services/ecuador/ecuadordata/MapServer?f=pjson
                # http://cga1.cga.harvard.edu/arcgis/services/ecuador/
                # ecuadordata/MapServer/WMSServer?request=GetCapabilities&service=WMS
                wms_url = service.url.replace('/rest/services/', '/services/')
                if '?f=pjson' in wms_url:
                    wms_url = wms_url.replace('?f=pjson', 'WMSServer?')
                if '?f=json' in wms_url:
                    wms_url = wms_url.replace('?f=json', 'WMSServer?')
                LOGGER.debug('This ESRI REST endpoint has an WMS interface to process: %s' % wms_url)
                # import here as otherwise is circular (TODO refactor)
                from utils import create_service_from_endpoint
                create_service_from_endpoint(wms_url, 'OGC:WMS', catalog=service.catalog)
        # now process the REST interface
        layer_n = 0
        total = len(esri_service.layers)
        for esri_layer in esri_service.layers:
            # in some case the json is invalid
            # esri_layer._json_struct
            # {u'currentVersion': 10.01,
            # u'error':
            # {u'message': u'An unexpected error occurred processing the request.', u'code': 500, u'details': []}}
            if 'error' not in esri_layer._json_struct:
                LOGGER.debug('Updating layer %s' % esri_layer.name)
                layer, created = Layer.objects.get_or_create(
                    name=esri_layer.id,
                    service=service,
                    catalog=service.catalog
                )
                if layer.active:
                    layer.type = 'ESRI:ArcGIS:MapServer'
                    links = [[layer.type, service.url],
                             ['OGC:WMTS', settings.SITE_URL.rstrip('/') + '/' + layer.get_url_endpoint()]]
                    layer.title = esri_layer.name
                    layer.abstract = esri_service.serviceDescription
                    layer.url = service.url
                    layer.page_url = layer.get_absolute_url
                    links.append([
                        'WWW:LINK',
                        settings.SITE_URL.rstrip('/') + layer.page_url
                    ])
                    try:
                        layer.bbox_x0 = esri_layer.extent.xmin
                        layer.bbox_y0 = esri_layer.extent.ymin
                        layer.bbox_x1 = esri_layer.extent.xmax
                        layer.bbox_y1 = esri_layer.extent.ymax
                    except KeyError:
                        pass
                    try:
                        layer.bbox_x0 = esri_layer._json_struct['extent']['xmin']
                        layer.bbox_y0 = esri_layer._json_struct['extent']['ymin']
                        layer.bbox_x1 = esri_layer._json_struct['extent']['xmax']
                        layer.bbox_y1 = esri_layer._json_struct['extent']['ymax']
                    except Exception:
                        pass
                    layer.wkt_geometry = bbox2wktpolygon([layer.bbox_x0, layer.bbox_y0, layer.bbox_x1, layer.bbox_y1])
                    layer.xml = create_metadata_record(
                        identifier=str(layer.uuid),
                        source=service.url,
                        links=links,
                        format='ESRI:ArcGIS:MapServer',
                        type=layer.csw_type,
                        relation=service.id_string,
                        title=layer.title,
                        alternative=layer.title,
                        abstract=layer.abstract,
                        wkt_geometry=layer.wkt_geometry
                    )
                    layer.anytext = gen_anytext(layer.title, layer.abstract)
                    layer.save()
                    # dates
                    add_mined_dates(layer)
                layer_n = layer_n + 1
                # exits if DEBUG_SERVICES
                LOGGER.debug("Updating layer n. %s/%s" % (layer_n, total))
                if DEBUG_SERVICES and layer_n == DEBUG_LAYER_NUMBER:
                    return
    except Exception as err:
        message = "update_layers_esri_mapserver: {0}".format(
            err
        )
        check = Check(
            content_object=service,
            success=False,
            response_time=0,
            message=message
        )
        check.save()


Instruction:
How would you explain what the following Python 3 function does
def update_layers_esri_imageserver(service):
    try:
        esri_service = ArcImageService(service.url)
        # set srs
        # both mapserver and imageserver exposes just one srs at the service level
        # not sure if other ones are supported, for now we just store this one
        obj = json.loads(esri_service._contents)
        srs_code = obj['spatialReference']['wkid']
        srs, created = SpatialReferenceSystem.objects.get_or_create(code=srs_code)
        service.srs.add(srs)

        service.update_validity()

        layer, created = Layer.objects.get_or_create(name=obj['name'], service=service, catalog=service.catalog)
        if layer.active:
            layer.type = 'ESRI:ArcGIS:ImageServer'
            links = [[layer.type, service.url],
                     ['OGC:WMTS', settings.SITE_URL.rstrip('/') + '/' + layer.get_url_endpoint()]]
            layer.title = obj['name']
            layer.abstract = esri_service.serviceDescription
            layer.url = service.url
            layer.bbox_x0 = str(obj['extent']['xmin'])
            layer.bbox_y0 = str(obj['extent']['ymin'])
            layer.bbox_x1 = str(obj['extent']['xmax'])
            layer.bbox_y1 = str(obj['extent']['ymax'])
            layer.page_url = layer.get_absolute_url
            links.append([
                'WWW:LINK',
                settings.SITE_URL.rstrip('/') + layer.page_url
            ])
            layer.wkt_geometry = bbox2wktpolygon([layer.bbox_x0, layer.bbox_y0, layer.bbox_x1, layer.bbox_y1])
            layer.xml = create_metadata_record(
                identifier=str(layer.uuid),
                source=service.url,
                links=links,
                format='ESRI:ArcGIS:ImageServer',
                type=layer.csw_type,
                relation=service.id_string,
                title=layer.title,
                alternative=layer.title,
                abstract=layer.abstract,
                wkt_geometry=layer.wkt_geometry
            )
            layer.anytext = gen_anytext(layer.title, layer.abstract)
            layer.save()
            # dates
            add_mined_dates(layer)
    except Exception as err:
        message = "update_layers_esri_imageserver: {0}".format(
            err
        )
        check = Check(
            content_object=service,
            success=False,
            response_time=0,
            message=message
        )
        check.save()

Response: Update layers for an ESRI REST ImageServer.


Instruction:
Can you generate the documentation for the following Python 3 function
def endpointlist_post_save(instance, *args, **kwargs):
    with open(instance.upload.file.name, mode='rb') as f:
        lines = f.readlines()
    for url in lines:
        if len(url) > 255:
            LOGGER.debug('Skipping this endpoint, as it is more than 255 characters: %s' % url)
        else:
            if Endpoint.objects.filter(url=url, catalog=instance.catalog).count() == 0:
                endpoint = Endpoint(url=url, endpoint_list=instance)
                endpoint.catalog = instance.catalog
                endpoint.save()
    if not settings.REGISTRY_SKIP_CELERY:
        update_endpoints.delay(instance.id)
    else:
        update_endpoints(instance.id)

Response: Called when the file is saved to the endpoint list.


Instruction:
Can you tell what is the following Python 3 function doing
def service_pre_save(instance, *args, **kwargs):

    # check if service is unique
    # we cannot use unique_together as it relies on a combination of fields
    # from different models (service, resource)
    exists = Service.objects.filter(url=instance.url,
                                    type=instance.type,
                                    catalog=instance.catalog).count() > 0

    # TODO: When saving from the django admin, this should not be triggered.
    # Reference: http://stackoverflow.com/questions/11561722/django-what-is-the-role-of-modelstate
    if instance._state.adding and exists:
        raise Exception("There is already such a service. url={0} catalog={1}".format(
            instance.url, instance.catalog
        ))

Response: Check if a service is unique and if not raise an exception


Instruction:
Can you generate the documentation for the following Python 3 function
def service_post_save(instance, *args, **kwargs):

    # check service
    if instance.is_monitored and settings.REGISTRY_SKIP_CELERY:
        check_service(instance.id)
    elif instance.is_monitored:
        check_service.delay(instance.id)

Response: Check if the service is full or not.


Instruction:
Can you implement a function in Python 3 that
checks validity of a WM layer.

Response:
def layer_pre_save(instance, *args, **kwargs):
    """
    Used to check layer validity.
    """

    is_valid = True

    # we do not need to check validity for WM layers
    if not instance.service.type == 'Hypermap:WorldMap':

        # 0. a layer is invalid if its service its invalid as well
        if not instance.service.is_valid:
            is_valid = False
            LOGGER.debug('Layer with id %s is marked invalid because its service is invalid' % instance.id)

        # 1. a layer is invalid with an extent within (-2, -2, +2, +2)
        if instance.bbox_x0 > -2 and instance.bbox_x1 < 2 and instance.bbox_y0 > -2 and instance.bbox_y1 < 2:
            is_valid = False
            LOGGER.debug(
                'Layer with id %s is marked invalid because its extent is within (-2, -2, +2, +2)' % instance.id
            )

    instance.is_valid = is_valid


Instruction:
Can you generate a brief explanation for the following Python 3 code
def layer_post_save(instance, *args, **kwargs):
    if instance.is_monitored and instance.service.is_monitored:  # index and monitor
        if not settings.REGISTRY_SKIP_CELERY:
            check_layer.delay(instance.id)
        else:
            check_layer(instance.id)
    else:  # just index
        index_layer(instance.id)

Response: Called when a layer is saved.


Instruction:
How would you code a function in Python 3 to
use to do reindex layers/services when a issue is removed form them.

Response:
def issue_post_delete(instance, *args, **kwargs):
    """
    Used to do reindex layers/services when a issue is removed form them.
    """
    LOGGER.debug('Re-adding layer/service to search engine index')
    if isinstance(instance.content_object, Service):
        if not settings.REGISTRY_SKIP_CELERY:
            index_service.delay(instance.content_object.id)
        else:
            index_service(instance.content_object.id)
    else:
        if not settings.REGISTRY_SKIP_CELERY:
            index_layer.delay(instance.content_object.id)
        else:
            index_layer(instance.content_object.id)


Instruction:
Write a Python 3 function for
returning the admin url with the check list for this instance.

Response:
def get_checks_admin_reliability_warning_url(self):
        """
        When service Realiability is going down users should go to the
        the check history to find problem causes.
        :return: admin url with check list for this instance
        """
        # TODO: cache this.
        path = self.get_checks_admin_url()
        content_type = ContentType.objects.get_for_model(self)
        params = "?content_type__id__exact={0}&q={1}&success__exact=0".format(
            content_type.id,
            self.id
        )
        url = path + params
        return url


Instruction:
Can you create a Python 3 function that
updates layers for a service.

Response:
def update_layers(self):
        """
        Update layers for a service.
        """

        signals.post_save.disconnect(layer_post_save, sender=Layer)

        try:
            LOGGER.debug('Updating layers for service id %s' % self.id)
            if self.type == 'OGC:WMS':
                update_layers_wms(self)
            elif self.type == 'OGC:WMTS':
                update_layers_wmts(self)
            elif self.type == 'ESRI:ArcGIS:MapServer':
                update_layers_esri_mapserver(self)
            elif self.type == 'ESRI:ArcGIS:ImageServer':
                update_layers_esri_imageserver(self)
            elif self.type == 'Hypermap:WorldMapLegacy':
                update_layers_wm_legacy(self)
            elif self.type == 'Hypermap:WorldMap':
                update_layers_geonode_wm(self)
            elif self.type == 'Hypermap:WARPER':
                update_layers_warper(self)

        except:
            LOGGER.error('Error updating layers for service %s' % self.uuid)

        signals.post_save.connect(layer_post_save, sender=Layer)


Instruction:
Write a Python 3 script for
checking availability of a service and provide run metrics.

Response:
def check_available(self):
        """
        Check for availability of a service and provide run metrics.
        """
        success = True
        start_time = datetime.datetime.utcnow()
        message = ''

        LOGGER.debug('Checking service id %s' % self.id)

        try:
            title = None
            abstract = None
            keywords = []
            wkt_geometry = None
            srs = '4326'
            if self.type == 'OGC:CSW':
                ows = CatalogueServiceWeb(self.url)
                title = ows.identification.title
                abstract = ows.identification.abstract
                keywords = ows.identification.keywords
            if self.type == 'OGC:WMS':
                ows = get_wms_version_negotiate(self.url)
                title = ows.identification.title
                abstract = ows.identification.abstract
                keywords = ows.identification.keywords
                for c in ows.contents:
                    if ows.contents[c].parent is None:
                        wkt_geometry = bbox2wktpolygon(ows.contents[c].boundingBoxWGS84)
                    break
            if self.type == 'OGC:WMTS':
                ows = WebMapTileService(self.url)
                title = ows.identification.title
                abstract = ows.identification.abstract
                keywords = ows.identification.keywords
            if self.type == 'OSGeo:TMS':
                ows = TileMapService(self.url)
                title = ows.identification.title
                abstract = ows.identification.abstract
                keywords = ows.identification.keywords
            if self.type == 'ESRI:ArcGIS:MapServer':
                esri = ArcMapService(self.url)
                extent, srs = get_esri_extent(esri)
                title = esri.mapName
                if len(title) == 0:
                    title = get_esri_service_name(self.url)
                wkt_geometry = bbox2wktpolygon([
                    extent['xmin'],
                    extent['ymin'],
                    extent['xmax'],
                    extent['ymax']
                ])
            if self.type == 'ESRI:ArcGIS:ImageServer':
                esri = ArcImageService(self.url)
                extent, srs = get_esri_extent(esri)
                title = esri._json_struct['name']
                if len(title) == 0:
                    title = get_esri_service_name(self.url)
                wkt_geometry = bbox2wktpolygon([
                    extent['xmin'],
                    extent['ymin'],
                    extent['xmax'],
                    extent['ymax']
                ])
            if self.type == 'Hypermap:WorldMap':
                urllib2.urlopen(self.url)
            if self.type == 'Hypermap:WorldMapLegacy':
                urllib2.urlopen(self.url)
                title = 'Harvard WorldMap Legacy'
            if self.type == 'Hypermap:WARPER':
                urllib2.urlopen(self.url)
            # update title without raising a signal and recursion
            if title:
                self.title = title
                Service.objects.filter(id=self.id).update(title=title)
            if abstract:
                self.abstract = abstract
                Service.objects.filter(id=self.id).update(abstract=abstract)
            if keywords:
                for kw in keywords:
                    # FIXME: persist keywords to Django model
                    self.keywords.add(kw)
            if wkt_geometry:
                self.wkt_geometry = wkt_geometry
                Service.objects.filter(id=self.id).update(wkt_geometry=wkt_geometry)
            xml = create_metadata_record(
                identifier=self.id_string,
                source=self.url,
                links=[[self.type, self.url]],
                format=self.type,
                type='service',
                title=title,
                abstract=abstract,
                keywords=keywords,
                wkt_geometry=self.wkt_geometry,
                srs=srs
            )
            anytexts = gen_anytext(title, abstract, keywords)
            Service.objects.filter(id=self.id).update(anytext=anytexts, xml=xml, csw_type='service')
        except Exception, e:
            LOGGER.error(e, exc_info=True)
            message = str(e)
            success = False

        end_time = datetime.datetime.utcnow()
        delta = end_time - start_time
        response_time = '%s.%s' % (delta.seconds, delta.microseconds)

        check = Check(
            content_object=self,
            success=success,
            response_time=response_time,
            message=message
        )
        check.save()
        LOGGER.debug('Service checked in %s seconds, status is %s' % (response_time, success))