Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_suitable_date_for_daily_extract(self, date=None, ut=False):
        daily_trips = self.get_trip_counts_per_day()
        max_daily_trips = daily_trips[u'trip_counts'].max(axis=0)
        if date in daily_trips[u'date_str']:
            start_index = daily_trips[daily_trips[u'date_str'] == date].index.tolist()[0]
            daily_trips[u'old_index'] = daily_trips.index
            daily_trips[u'date_dist'] = abs(start_index - daily_trips.index)
            daily_trips = daily_trips.sort_values(by=[u'date_dist', u'old_index']).reindex()
        for row in daily_trips.itertuples():
            if row.trip_counts >= 0.9 * max_daily_trips:
                if ut:
                    return self.get_day_start_ut(row.date_str)
                else:
                    return row.date_str

Response: Returns the suitable date for daily extract.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_weekly_extract_start_date(self, ut=False, weekdays_at_least_of_max=0.9,
                                      verbose=False, download_date_override=None):
        """
        Find a suitable weekly extract start date (monday).
        The goal is to obtain as 'usual' week as possible.
        The weekdays of the weekly extract week should contain
        at least 0.9 of the total maximum of trips.

        Parameters
        ----------
        ut: return unixtime?
        weekdays_at_least_of_max: float

        download_date_override: str, semi-optional
            Download-date in format %Y-%m-%d, weeks close to this.
            Overrides the (possibly) recorded downloaded date in the database

        Returns
        -------
        date: int or str

        Raises
        ------
        error: RuntimeError
            If no download date could be found.
        """
        daily_trip_counts = self.get_trip_counts_per_day()
        if isinstance(download_date_override, str):
            search_start_date = datetime.datetime.strptime(download_date_override, "%Y-%m-%d")
        elif isinstance(download_date_override, datetime.datetime):
            search_start_date = download_date_override
        else:
            assert download_date_override is None
            download_date_str = self.meta['download_date']
            if download_date_str == "":
                warnings.warn("Download date is not speficied in the database. "
                              "Download date used in GTFS." + self.get_weekly_extract_start_date.__name__ +
                              "() defaults to the smallest date when any operations take place.")
                search_start_date = daily_trip_counts['date'].min()
            else:
                search_start_date = datetime.datetime.strptime(download_date_str, "%Y-%m-%d")

        feed_min_date = daily_trip_counts['date'].min()
        feed_max_date = daily_trip_counts['date'].max()
        assert (feed_max_date - feed_min_date >= datetime.timedelta(days=7)), \
            "Dataset is not long enough for providing week long extracts"

        # get first a valid monday where the search for the week can be started:
        next_monday_from_search_start_date = search_start_date + timedelta(days=(7 - search_start_date.weekday()))
        if not (feed_min_date <= next_monday_from_search_start_date <= feed_max_date):
            warnings.warn("The next monday after the (possibly user) specified download date is not present in the database."
                          "Resorting to first monday after the beginning of operations instead.")
            next_monday_from_search_start_date = feed_min_date + timedelta(days=(7 - feed_min_date.weekday()))

        max_trip_count = daily_trip_counts['trip_counts'].quantile(0.95)
        # Take 95th percentile to omit special days, if any exist.

        threshold = weekdays_at_least_of_max * max_trip_count
        threshold_fulfilling_days = daily_trip_counts['trip_counts'] > threshold

        # look forward first
        # get the index of the trip:
        search_start_monday_index = daily_trip_counts[daily_trip_counts['date'] == next_monday_from_search_start_date].index[0]

        # get starting point
        while_loop_monday_index = search_start_monday_index
        while len(daily_trip_counts.index) >= while_loop_monday_index + 7:
            if all(threshold_fulfilling_days[while_loop_monday_index:while_loop_monday_index + 5]):
                row = daily_trip_counts.iloc[while_loop_monday_index]
                if ut:
                    return self.get_day_start_ut(row.date_str)
                else:
                    return row['date']
            while_loop_monday_index += 7

        while_loop_monday_index = search_start_monday_index - 7
        # then backwards
        while while_loop_monday_index >= 0:
            if all(threshold_fulfilling_days[while_loop_monday_index:while_loop_monday_index + 5]):
                row = daily_trip_counts.iloc[while_loop_monday_index]
                if ut:
                    return self.get_day_start_ut(row.date_str)
                else:
                    return row['date']
            while_loop_monday_index -= 7

        raise RuntimeError("No suitable weekly extract start date could be determined!")

Response: Returns a date that can be used to obtain the start date of the weekly extract week.


Instruction:
Can you tell what is the following Python 3 function doing
def get_closest_stop(self, lat, lon):
        cur = self.conn.cursor()
        min_dist = float("inf")
        min_stop_I = None
        rows = cur.execute("SELECT stop_I, lat, lon FROM stops")
        for stop_I, lat_s, lon_s in rows:
            dist_now = wgs84_distance(lat, lon, lat_s, lon_s)
            if dist_now < min_dist:
                min_dist = dist_now
                min_stop_I = stop_I
        return min_stop_I

Response: Get the closest stop to a given location.


Instruction:
Explain what the following Python 3 code does
def get_route_name_and_type_of_tripI(self, trip_I):
        cur = self.conn.cursor()
        results = cur.execute("SELECT name, type FROM routes JOIN trips USING(route_I) WHERE trip_I={trip_I}"
                              .format(trip_I=trip_I))
        name, rtype = results.fetchone()
        return u"%s" % str(name), int(rtype)

Response: Get route short name and type of a given trip.


Instruction:
Create a Python 3 function to
get route short name and type of a route.

Response:
def get_route_name_and_type(self, route_I):
        """
        Get route short name and type

        Parameters
        ----------
        route_I: int
            route index (database specific)

        Returns
        -------
        name: str
            short name of the route, eg. 195N
        type: int
            route_type according to the GTFS standard
        """
        cur = self.conn.cursor()
        results = cur.execute("SELECT name, type FROM routes WHERE route_I=(?)", (route_I,))
        name, rtype = results.fetchone()
        return name, int(rtype)


Instruction:
How would you explain what the following Python 3 function does
def get_trip_stop_coordinates(self, trip_I):
        query = """SELECT lat, lon
                    FROM stop_times
                    JOIN stops
                    USING(stop_I)
                        WHERE trip_I={trip_I}
                    ORDER BY stop_times.seq""".format(trip_I=trip_I)
        stop_coords = pd.read_sql(query, self.conn)
        return stop_coords

Response: Get the stop coordinates for a given trip_I


Instruction:
Given the following Python 3 function, write the documentation
def get_trip_stop_time_data(self, trip_I, day_start_ut):
        to_select = "stop_I, " + str(day_start_ut) + "+dep_time_ds AS dep_time_ut, lat, lon, seq, shape_break"
        str_to_run = "SELECT " + to_select + """
                        FROM stop_times JOIN stops USING(stop_I)
                        WHERE (trip_I ={trip_I}) ORDER BY seq
                      """
        str_to_run = str_to_run.format(trip_I=trip_I)
        return pd.read_sql_query(str_to_run, self.conn)

Response: Get the data for a particular trip stop


Instruction:
Write a Python 3 function for
getting the events that are in the given trip and time range.

Response:
def get_events_by_tripI_and_dsut(self, trip_I, day_start_ut,
                                     start_ut=None, end_ut=None):
        """
        Get trip data as a list of events (i.e. dicts).

        Parameters
        ----------
        trip_I : int
            shorthand index of the trip.
        day_start_ut : int
            the start time of the day in unix time (seconds)
        start_ut : int, optional
            consider only events that start after this time
            If not specified, this filtering is not applied.
        end_ut : int, optional
            Consider only events that end before this time
            If not specified, this filtering is not applied.

        Returns
        -------
        events: list of dicts
            each element contains the following data:
                from_stop: int (stop_I)
                to_stop: int (stop_I)
                dep_time_ut: int (in unix time)
                arr_time_ut: int (in unix time)
        """
        # for checking input:
        assert day_start_ut <= start_ut
        assert day_start_ut <= end_ut
        assert start_ut <= end_ut
        events = []
        # check that trip takes place on that day:
        if not self.tripI_takes_place_on_dsut(trip_I, day_start_ut):
            return events

        query = """SELECT stop_I, arr_time_ds+?, dep_time_ds+?
                    FROM stop_times JOIN stops USING(stop_I)
                    WHERE
                        (trip_I = ?)
                """
        params = [day_start_ut, day_start_ut,
                  trip_I]
        if start_ut:
            query += "AND (dep_time_ds > ?-?)"
            params += [start_ut, day_start_ut]
        if end_ut:
            query += "AND (arr_time_ds < ?-?)"
            params += [end_ut, day_start_ut]
        query += "ORDER BY arr_time_ds"
        cur = self.conn.cursor()
        rows = cur.execute(query, params)
        stop_data = list(rows)
        for i in range(len(stop_data) - 1):
            event = {
                "from_stop": stop_data[i][0],
                "to_stop": stop_data[i + 1][0],
                "dep_time_ut": stop_data[i][2],
                "arr_time_ut": stop_data[i + 1][1]
            }
            events.append(event)
        return events


Instruction:
Can you write a function in Python 3 where it
checks that a trip takes place during a given day

Response:
def tripI_takes_place_on_dsut(self, trip_I, day_start_ut):
        """
        Check that a trip takes place during a day

        Parameters
        ----------
        trip_I : int
            index of the trip in the gtfs data base
        day_start_ut : int
            the starting time of the day in unix time (seconds)

        Returns
        -------
        takes_place: bool
            boolean value describing whether the trip takes place during
            the given day or not
        """
        query = "SELECT * FROM days WHERE trip_I=? AND day_start_ut=?"
        params = (trip_I, day_start_ut)
        cur = self.conn.cursor()
        rows = list(cur.execute(query, params))
        if len(rows) == 0:
            return False
        else:
            assert len(rows) == 1, 'On a day, a trip_I should be present at most once'
            return True


Instruction:
Make a summary of the following Python 3 code
def day_start_ut(self, ut):
        # set timezone to the one of gtfs
        old_tz = self.set_current_process_time_zone()
        ut = time.mktime(time.localtime(ut)[:3] + (12, 00, 0, 0, 0, -1)) - 43200
        set_process_timezone(old_tz)
        return ut

Response: Convert unixtime to unixtime on GTFS start - of - day.


Instruction:
How would you code a function in Python 3 to
increment the GTFS - definition of day start.

Response:
def increment_day_start_ut(self, day_start_ut, n_days=1):
        """Increment the GTFS-definition of "day start".

        Parameters
        ----------
        day_start_ut : int
            unixtime of the previous start of day.  If this time is between
            12:00 or greater, there *will* be bugs.  To solve this, run the
            input through day_start_ut first.
        n_days: int
            number of days to increment
        """
        old_tz = self.set_current_process_time_zone()
        day0 = time.localtime(day_start_ut + 43200)  # time of noon
        dayN = time.mktime(day0[:2] +  # YYYY, MM
                           (day0[2] + n_days,) +  # DD
                           (12, 00, 0, 0, 0, -1)) - 43200  # HHMM, etc.  Minus 12 hours.
        set_process_timezone(old_tz)
        return dayN


Instruction:
Can you tell what is the following Python 3 function doing
def _get_possible_day_starts(self, start_ut, end_ut, max_time_overnight=None):
        if max_time_overnight is None:
            # 7 hours:
            max_time_overnight = 7 * 60 * 60

        # sanity checks for the timezone parameter
        # assert timezone < 14
        # assert timezone > -14
        # tz_seconds = int(timezone*3600)
        assert start_ut < end_ut
        start_day_ut = self.day_start_ut(start_ut)
        # start_day_ds = int(start_ut+tz_seconds) % seconds_in_a_day  #??? needed?
        start_day_ds = start_ut - start_day_ut
        # assert (start_day_ut+tz_seconds) % seconds_in_a_day == 0
        end_day_ut = self.day_start_ut(end_ut)
        # end_day_ds = int(end_ut+tz_seconds) % seconds_in_a_day    #??? needed?
        # end_day_ds = end_ut - end_day_ut
        # assert (end_day_ut+tz_seconds) % seconds_in_a_day == 0

        # If we are early enough in a day that we might have trips from
        # the previous day still running, decrement the start day.
        if start_day_ds < max_time_overnight:
            start_day_ut = self.increment_day_start_ut(start_day_ut, n_days=-1)

        # day_start_times_ut = range(start_day_ut, end_day_ut+seconds_in_a_day, seconds_in_a_day)

        # Create a list of all possible day start times.  This is roughly
        # range(day_start_ut, day_end_ut+1day, 1day).
        day_start_times_ut = [start_day_ut]
        while day_start_times_ut[-1] < end_day_ut:
            day_start_times_ut.append(self.increment_day_start_ut(day_start_times_ut[-1]))

        start_times_ds = []
        end_times_ds = []
        # For every possible day start:
        for dsut in day_start_times_ut:
            # start day_seconds starts at either zero, or time - daystart
            day_start_ut = max(0, start_ut - dsut)
            start_times_ds.append(day_start_ut)
            # end day_seconds is time-day_start
            day_end_ut = end_ut - dsut
            end_times_ds.append(day_end_ut)
        # Return three tuples which can be zip:ped together.
        return day_start_times_ut, start_times_ds, end_times_ds

Response: Get all possible day start times between start_ut and end_ut


Instruction:
Explain what the following Python 3 code does
def get_tripIs_within_range_by_dsut(self,
                                        start_time_ut,
                                        end_time_ut):
        """
        Obtain a list of trip_Is that take place during a time interval.
        The trip needs to be only partially overlapping with the given time interval.
        The grouping by dsut (day_start_ut) is required as same trip_I could
        take place on multiple days.

        Parameters
        ----------
        start_time_ut : int
            start of the time interval in unix time (seconds)
        end_time_ut: int
            end of the time interval in unix time (seconds)

        Returns
        -------
        trip_I_dict: dict
            keys: day_start_times to list of integers (trip_Is)
        """
        cur = self.conn.cursor()
        assert start_time_ut <= end_time_ut
        dst_ut, st_ds, et_ds = \
            self._get_possible_day_starts(start_time_ut, end_time_ut, 7)
        # noinspection PyTypeChecker
        assert len(dst_ut) >= 0
        trip_I_dict = {}
        for day_start_ut, start_ds, end_ds in \
                zip(dst_ut, st_ds, et_ds):
            query = """
                        SELECT distinct(trip_I)
                        FROM days
                            JOIN trips
                            USING(trip_I)
                        WHERE
                            (days.day_start_ut == ?)
                            AND (
                                    (trips.start_time_ds <= ?)
                                    AND
                                    (trips.end_time_ds >= ?)
                                )
                        """
            params = (day_start_ut, end_ds, start_ds)
            trip_Is = [el[0] for el in cur.execute(query, params)]
            if len(trip_Is) > 0:
                trip_I_dict[day_start_ut] = trip_Is
        return trip_I_dict

Response: Returns a dict of trip_Is that take place during a given time interval.


Instruction:
Create a Python 3 function for
getting all stop data as a pandas DataFrame for all stops or an individual stop

Response:
def stop(self, stop_I):
        """
        Get all stop data as a pandas DataFrame for all stops, or an individual stop'

        Parameters
        ----------
        stop_I : int
            stop index

        Returns
        -------
        stop: pandas.DataFrame
        """
        return pd.read_sql_query("SELECT * FROM stops WHERE stop_I={stop_I}".format(stop_I=stop_I), self.conn)


Instruction:
Can you implement a function in Python 3 that
returns a DataFrame containing all stops for a given route type.

Response:
def get_stops_for_route_type(self, route_type):
        """
        Parameters
        ----------
        route_type: int

        Returns
        -------
        stops: pandas.DataFrame

        """
        if route_type is WALK:
            return self.stops()
        else:
            return pd.read_sql_query("SELECT DISTINCT stops.* "
                                     "FROM stops JOIN stop_times ON stops.stop_I == stop_times.stop_I "
                                     "           JOIN trips ON stop_times.trip_I = trips.trip_I"
                                     "           JOIN routes ON trips.route_I == routes.route_I "
                                     "WHERE routes.type=(?)", self.conn, params=(route_type,))


Instruction:
Can you tell what is the following Python 3 function doing
def generate_routable_transit_events(self, start_time_ut=None, end_time_ut=None, route_type=None):
        from gtfspy.networks import temporal_network
        df = temporal_network(self, start_time_ut=start_time_ut, end_time_ut=end_time_ut, route_type=route_type)
        df.sort_values("dep_time_ut", ascending=False, inplace=True)

        for row in df.itertuples():
            yield row

Response: Generate a list of tuples that define a transition between start_time_ut and end_time_ut.


Instruction:
Make a summary of the following Python 3 code
def get_transit_events(self, start_time_ut=None, end_time_ut=None, route_type=None):
        table_name = self._get_day_trips_table_name()
        event_query = "SELECT stop_I, seq, trip_I, route_I, routes.route_id AS route_id, routes.type AS route_type, " \
                      "shape_id, day_start_ut+dep_time_ds AS dep_time_ut, day_start_ut+arr_time_ds AS arr_time_ut " \
                      "FROM " + table_name + " " \
                                             "JOIN trips USING(trip_I) " \
                                             "JOIN routes USING(route_I) " \
                                             "JOIN stop_times USING(trip_I)"

        where_clauses = []
        if end_time_ut:
            where_clauses.append(table_name + ".start_time_ut< {end_time_ut}".format(end_time_ut=end_time_ut))
            where_clauses.append("dep_time_ut  <={end_time_ut}".format(end_time_ut=end_time_ut))
        if start_time_ut:
            where_clauses.append(table_name + ".end_time_ut  > {start_time_ut}".format(start_time_ut=start_time_ut))
            where_clauses.append("arr_time_ut  >={start_time_ut}".format(start_time_ut=start_time_ut))
        if route_type is not None:
            assert route_type in ALL_ROUTE_TYPES
            where_clauses.append("routes.type={route_type}".format(route_type=route_type))
        if len(where_clauses) > 0:
            event_query += " WHERE "
            for i, where_clause in enumerate(where_clauses):
                if i is not 0:
                    event_query += " AND "
                event_query += where_clause
        # ordering is required for later stages
        event_query += " ORDER BY trip_I, day_start_ut+dep_time_ds;"
        events_result = pd.read_sql_query(event_query, self.conn)
        # 'filter' results so that only real "events" are taken into account
        from_indices = numpy.nonzero(
            (events_result['trip_I'][:-1].values == events_result['trip_I'][1:].values) *
            (events_result['seq'][:-1].values < events_result['seq'][1:].values)
        )[0]
        to_indices = from_indices + 1
        # these should have same trip_ids
        assert (events_result['trip_I'][from_indices].values == events_result['trip_I'][to_indices].values).all()
        trip_Is = events_result['trip_I'][from_indices]
        from_stops = events_result['stop_I'][from_indices]
        to_stops = events_result['stop_I'][to_indices]
        shape_ids = events_result['shape_id'][from_indices]
        dep_times = events_result['dep_time_ut'][from_indices]
        arr_times = events_result['arr_time_ut'][to_indices]
        route_types = events_result['route_type'][from_indices]
        route_ids = events_result['route_id'][from_indices]
        route_Is = events_result['route_I'][from_indices]
        durations = arr_times.values - dep_times.values
        assert (durations >= 0).all()
        from_seqs = events_result['seq'][from_indices]
        to_seqs = events_result['seq'][to_indices]
        data_tuples = zip(from_stops, to_stops, dep_times, arr_times,
                          shape_ids, route_types, route_ids, trip_Is,
                          durations, from_seqs, to_seqs, route_Is)
        columns = ["from_stop_I", "to_stop_I", "dep_time_ut", "arr_time_ut",
                   "shape_id", "route_type", "route_id", "trip_I",
                   "duration", "from_seq", "to_seq", "route_I"]
        df = pd.DataFrame.from_records(data_tuples, columns=columns)
        return df

Response: Returns a pandas. DataFrame containing all events that take place during a given time interval.


Instruction:
Write a Python 3 function for
returning the difference between the routes in this db and the ones in the other db.

Response:
def get_route_difference_with_other_db(self, other_gtfs, start_time, end_time, uniqueness_threshold=None,
                                           uniqueness_ratio=None):
        """
        Compares the routes based on stops in the schedule with the routes in another db and returns the ones without match.
        Uniqueness thresholds or ratio can be used to allow small differences
        :param uniqueness_threshold:
        :param uniqueness_ratio:
        :return:
        """
        from gtfspy.stats import frequencies_by_generated_route

        this_df = frequencies_by_generated_route(self, start_time, end_time)
        other_df = frequencies_by_generated_route(other_gtfs, start_time, end_time)
        this_routes = {x: set(x.split(',')) for x in this_df["route"]}
        other_routes = {x: set(x.split(',')) for x in other_df["route"]}
        # this_df["route_set"] = this_df.apply(lambda x: set(x.route.split(',')), axis=1)
        # other_df["route_set"] = other_df.apply(lambda x: set(x.route.split(',')), axis=1)

        this_uniques = list(this_routes.keys())
        other_uniques = list(other_routes.keys())
        print("initial routes A:", len(this_uniques))
        print("initial routes B:", len(other_uniques))
        for i_key, i in this_routes.items():
            for j_key, j in other_routes.items():
                union = i | j
                intersection = i & j
                symmetric_difference = i ^ j
                if uniqueness_ratio:
                    if len(intersection) / len(union) >= uniqueness_ratio:
                        try:
                            this_uniques.remove(i_key)
                            this_df = this_df[this_df["route"] != i_key]
                        except ValueError:
                            pass
                        try:
                            other_uniques.remove(j_key)
                            other_df = other_df[other_df["route"] != j_key]
                        except ValueError:
                            pass

        print("unique routes A", len(this_df))
        print("unique routes B", len(other_df))
        return this_df, other_df


Instruction:
Create a Python 3 function for
getting straight line transfer distances to stations that can be transferred to.

Response:
def get_straight_line_transfer_distances(self, stop_I=None):
        """
        Get (straight line) distances to stations that can be transferred to.

        Parameters
        ----------
        stop_I : int, optional
            If not specified return all possible transfer distances

        Returns
        -------
        distances: pandas.DataFrame
            each row has the following items
                from_stop_I: int
                to_stop_I: int
                d: float or int #distance in meters
        """
        if stop_I is not None:
            query = u""" SELECT from_stop_I, to_stop_I, d
                        FROM stop_distances
                            WHERE
                                from_stop_I=?
                    """
            params = (u"{stop_I}".format(stop_I=stop_I),)
        else:
            query = """ SELECT from_stop_I, to_stop_I, d
                        FROM stop_distances
                    """
            params = None
        stop_data_df = pd.read_sql_query(query, self.conn, params=params)
        return stop_data_df


Instruction:
Write a Python 3 function for
returning the first and last day_start_ut span of the last day_start_ut.

Response:
def get_day_start_ut_span(self):
        """
        Return the first and last day_start_ut

        Returns
        -------
        first_day_start_ut: int
        last_day_start_ut: int
        """
        cur = self.conn.cursor()
        first_day_start_ut, last_day_start_ut = \
            cur.execute("SELECT min(day_start_ut), max(day_start_ut) FROM days;").fetchone()
        return first_day_start_ut, last_day_start_ut


Instruction:
Can you generate the documentation for the following Python 3 function
def homogenize_stops_table_with_other_db(self, source):
        cur = self.conn.cursor()
        self.attach_gtfs_database(source)

        query_inner_join = """SELECT t1.*
                              FROM stops t1
                              INNER JOIN other.stops t2
                              ON t1.stop_id=t2.stop_id
                              AND find_distance(t1.lon, t1.lat, t2.lon, t2.lat) <= 50"""
        df_inner_join = self.execute_custom_query_pandas(query_inner_join)
        print("number of common stops: ", len(df_inner_join.index))
        df_not_in_other = self.execute_custom_query_pandas("SELECT * FROM stops EXCEPT " + query_inner_join)
        print("number of stops missing in second feed: ", len(df_not_in_other.index))
        df_not_in_self = self.execute_custom_query_pandas("SELECT * FROM other.stops EXCEPT " +
                                                          query_inner_join.replace("t1.*", "t2.*"))
        print("number of stops missing in first feed: ", len(df_not_in_self.index))
        try:
            self.execute_custom_query("""ALTER TABLE stops ADD COLUMN stop_pair_I INT """)

            self.execute_custom_query("""ALTER TABLE other.stops ADD COLUMN stop_pair_I INT """)
        except sqlite3.OperationalError:
            pass
        stop_id_stub = "added_stop_"
        counter = 0
        rows_to_update_self = []
        rows_to_update_other = []
        rows_to_add_to_self = []
        rows_to_add_to_other = []

        for items in df_inner_join.itertuples(index=False):
            rows_to_update_self.append((counter, items[1]))
            rows_to_update_other.append((counter, items[1]))
            counter += 1

        for items in df_not_in_other.itertuples(index=False):
            rows_to_update_self.append((counter, items[1]))
            rows_to_add_to_other.append((stop_id_stub + str(counter),) + tuple(items[x] for x in [2, 3, 4, 5, 6, 8, 9])
                                        + (counter,))
            counter += 1

        for items in df_not_in_self.itertuples(index=False):
            rows_to_update_other.append((counter, items[1]))
            rows_to_add_to_self.append((stop_id_stub + str(counter),) + tuple(items[x] for x in [2, 3, 4, 5, 6, 8, 9])
                                       + (counter,))
            counter += 1

        query_add_row = """INSERT INTO stops(
                                    stop_id,
                                    code,
                                    name,
                                    desc,
                                    lat,
                                    lon,
                                    location_type,
                                    wheelchair_boarding,
                                    stop_pair_I) VALUES (%s) """ % (", ".join(["?" for x in range(9)]))

        query_update_row = """UPDATE stops SET stop_pair_I=? WHERE stop_id=?"""
        print("adding rows to databases")
        cur.executemany(query_add_row, rows_to_add_to_self)
        cur.executemany(query_update_row, rows_to_update_self)
        cur.executemany(query_add_row.replace("stops", "other.stops"), rows_to_add_to_other)
        cur.executemany(query_update_row.replace("stops", "other.stops"), rows_to_update_other)
        self.conn.commit()
        print("finished")

Response: This function checks the external database for common stops and adds the missing stops to the common stops table.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def read_data_as_dataframe(self,
                               travel_impedance_measure,
                               from_stop_I=None,
                               to_stop_I=None,
                               statistic=None):
        """
        Recover pre-computed travel_impedance between od-pairs from the database.

        Returns
        -------
        values: number | Pandas DataFrame
        """
        to_select = []
        where_clauses = []
        to_select.append("from_stop_I")
        to_select.append("to_stop_I")
        if from_stop_I is not None:
            where_clauses.append("from_stop_I=" + str(int(from_stop_I)))
        if to_stop_I is not None:
            where_clauses.append("to_stop_I=" + str(int(to_stop_I)))
        where_clause = ""
        if len(where_clauses) > 0:
            where_clause = " WHERE " + " AND ".join(where_clauses)
        if not statistic:
            to_select.extend(["min", "mean", "median", "max"])
        else:
            to_select.append(statistic)
        to_select_clause = ",".join(to_select)
        if not to_select_clause:
            to_select_clause = "*"
        sql = "SELECT " + to_select_clause + " FROM " + travel_impedance_measure + where_clause + ";"
        df = pd.read_sql(sql, self.conn)
        return df

Response: Reads the data from the database into a Pandas DataFrame.


Instruction:
Here you have a function in Python 3, explain what it does
def insert_data(self, travel_impedance_measure_name, data):
        f = float
        data_tuple = [(int(x["from_stop_I"]), int(x["to_stop_I"]), f(x["min"]), f(x["max"]), f(x["median"]), f(x["mean"])) for
                      x in data]
        insert_stmt = '''INSERT OR REPLACE INTO ''' + travel_impedance_measure_name + ''' (
                              from_stop_I,
                              to_stop_I,
                              min,
                              max,
                              median,
                              mean) VALUES (?, ?, ?, ?, ?, ?) '''
        self.conn.executemany(insert_stmt, data_tuple)
        self.conn.commit()

Response: Inserts data into the table with the given name.


Instruction:
Make a summary of the following Python 3 code
def add_event(self, event):
        assert event.dep_time_ut <= event.arr_time_ut
        heappush(self.heap, event)

Response: Adds an event to the heap


Instruction:
How would you implement a function in Python 3 that
adds events from a given distance table to the heap.

Response:
def add_walk_events_to_heap(self, transfer_distances, e, start_time_ut, walk_speed, uninfected_stops, max_duration_ut):
        """
        Parameters
        ----------
        transfer_distances:
        e : Event
        start_time_ut : int
        walk_speed : float
        uninfected_stops : list
        max_duration_ut : int
        """
        n = len(transfer_distances)
        dists_values = transfer_distances.values
        to_stop_I_index = np.nonzero(transfer_distances.columns == 'to_stop_I')[0][0]
        d_index = np.nonzero(transfer_distances.columns == 'd')[0][0]
        for i in range(n):
            transfer_to_stop_I = dists_values[i, to_stop_I_index]
            if transfer_to_stop_I in uninfected_stops:
                d = dists_values[i, d_index]
                transfer_arr_time = e.arr_time_ut + int(d/float(walk_speed))
                if transfer_arr_time > start_time_ut+max_duration_ut:
                    continue
                te = Event(transfer_arr_time, e.arr_time_ut, e.to_stop_I, transfer_to_stop_I, WALK)
                self.add_event(te)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _check_dep_time_is_valid(self, dep_time):
        assert dep_time <= self._min_dep_time, "Labels should be entered in decreasing order of departure time."
        dep_time_index = self.dep_times_to_index[dep_time]
        if self._min_dep_time < float('inf'):
            min_dep_index = self.dep_times_to_index[self._min_dep_time]
            assert min_dep_index == dep_time_index or (min_dep_index == dep_time_index - 1), \
                "dep times should be ordered sequentially"
        else:
            assert dep_time_index is 0, "first dep_time index should be zero (ensuring that all connections are properly handled)"
        self._min_dep_time = dep_time

Response: A simple checker that the dep_time is valid.


Instruction:
Can you generate the documentation for the following Python 3 function
def update(self, new_labels, departure_time_backup=None):
        if self._closed:
            raise RuntimeError("Profile is closed, no updates can be made")
        try:
            departure_time = next(iter(new_labels)).departure_time
        except StopIteration:
            departure_time = departure_time_backup
        self._check_dep_time_is_valid(departure_time)

        for new_label in new_labels:
            assert (new_label.departure_time == departure_time)
        dep_time_index = self.dep_times_to_index[departure_time]

        if dep_time_index > 0:
            # Departure time is modified in order to not pass on labels which are not Pareto-optimal when departure time is ignored.
            mod_prev_labels = [label.get_copy_with_specified_departure_time(departure_time) for label
                               in self._label_bags[dep_time_index - 1]]
        else:
            mod_prev_labels = list()
        mod_prev_labels += self._label_bags[dep_time_index]

        walk_label = self._get_label_to_target(departure_time)
        if walk_label:
            new_labels = new_labels + [walk_label]
        new_frontier = merge_pareto_frontiers(new_labels, mod_prev_labels)

        self._label_bags[dep_time_index] = new_frontier
        return True

Response: Update the profile with the new labels.


Instruction:
Write a Python 3 script for
evaluating the pareto optimal set of labels given a departure time.

Response:
def evaluate(self, dep_time, first_leg_can_be_walk=True, connection_arrival_time=None):

        """
        Get the pareto_optimal set of Labels, given a departure time.

        Parameters
        ----------
        dep_time : float, int
            time in unix seconds
        first_leg_can_be_walk : bool, optional
            whether to allow walking to target to be included into the profile
            (I.e. whether this function is called when scanning a pseudo-connection:
            "double" walks are not allowed.)
        connection_arrival_time: float, int, optional
            used for computing the walking label if dep_time, i.e., connection.arrival_stop_next_departure_time, is infinity)
        connection: connection object

        Returns
        -------
        pareto_optimal_labels : set
            Set of Labels
        """
        walk_labels = list()
        # walk label towards target
        if first_leg_can_be_walk and self._walk_to_target_duration != float('inf'):
            # add walk_label
            if connection_arrival_time is not None:
                walk_labels.append(self._get_label_to_target(connection_arrival_time))
            else:
                walk_labels.append(self._get_label_to_target(dep_time))

        # if dep time is larger than the largest dep time -> only walk labels are possible
        if dep_time in self.dep_times_to_index:
            assert (dep_time != float('inf'))
            index = self.dep_times_to_index[dep_time]
            labels = self._label_bags[index]
            pareto_optimal_labels = merge_pareto_frontiers(labels, walk_labels)
        else:
            pareto_optimal_labels = walk_labels

        if not first_leg_can_be_walk:
            pareto_optimal_labels = [label for label in pareto_optimal_labels if not label.first_leg_is_walk]
        return pareto_optimal_labels


Instruction:
Given the following Python 3 function, write the documentation
def finalize(self, neighbor_label_bags=None, walk_durations=None, departure_arrival_stop_pairs=None):
        assert (not self._finalized)
        if self._final_pareto_optimal_labels is None:
            self._compute_real_connection_labels()
        if neighbor_label_bags is not None:
            assert (len(walk_durations) == len(neighbor_label_bags))
            self._compute_final_pareto_optimal_labels(neighbor_label_bags,
                                                      walk_durations,
                                                      departure_arrival_stop_pairs)
        else:
            self._final_pareto_optimal_labels = self._real_connection_labels
        self._finalized = True
        self._closed = True

Response: Finalizes the internal state of the internal state of the internal state of the internal state.


Instruction:
Create a Python 3 function to
do this GTFS contain this file? ( file specified by the class )

Response:
def exists_by_source(self):
        """Does this GTFS contain this file? (file specified by the class)"""
        exists_list = []
        for source in self.gtfs_sources:
            if isinstance(source, dict):
                # source can now be either a dict or a zipfile
                if self.fname in source:
                    if source[self.fname]:
                        exists_list.append(True)
                        continue
            # Handle zipfiles specially
            if "zipfile" in source:
                try:
                    Z = zipfile.ZipFile(source['zipfile'], mode='r')
                    Z.getinfo(os.path.join(source['zip_commonprefix'], self.fname))
                    exists_list.append(True)
                    continue
                # File does not exist in the zip archive
                except KeyError:
                    print(self.fname, ' missing in ', source)
                    exists_list.append(False)
                    continue
            # Normal filename
            elif isinstance(source, string_types):
                if os.path.exists(os.path.join(source, self.fname)):
                    exists_list.append(True)
                    continue
            exists_list.append(False)
        # the "file" was not found in any of the sources, return false
        return exists_list


Instruction:
Can you tell what is the following Python 3 function doing
def create_table(self, conn):
        # Make cursor
        cur = conn.cursor()
        # Drop table if it already exists, to be recreated.  This
        # could in the future abort if table already exists, and not
        # recreate it from scratch.
        #cur.execute('''DROP TABLE IF EXISTS %s'''%self.table)
        #conn.commit()
        if self.tabledef is None:
            return
        if not self.tabledef.startswith('CREATE'):
            # "normal" table creation.
            cur.execute('CREATE TABLE IF NOT EXISTS %s %s'
                        % (self.table, self.tabledef)
                        )
        else:
            # When tabledef contains the full CREATE statement (for
            # virtual tables).
            cur.execute(self.tabledef)
        conn.commit()

Response: Create the table if it does not exist.


Instruction:
Write a Python 3 script for
loading data from GTFS file into database

Response:
def insert_data(self, conn):
        """Load data from GTFS file into database"""
        cur = conn.cursor()
        # This is a bit hackish.  It is annoying to have to write the
        # INSERT statement yourself and keep it up to date with the
        # table rows.  This gets the first row, figures out the field
        # names from that, and then makes an INSERT statement like
        # "INSERT INTO table (col1, col2, ...) VALUES (:col1, :col2,
        # ...)".  The ":col1" is sqlite syntax for named value.

        csv_reader_generators, prefixes = self._get_csv_reader_generators()
        for csv_reader, prefix in zip(csv_reader_generators, prefixes):
            try:
                row = next(iter(self.gen_rows([csv_reader], [prefix])))
                fields = row.keys()
            except StopIteration:
                # The file has *only* a header and no data.
                # next(iter()) yields StopIteration and we can't
                # proceed.  Since there is nothing to import, just continue the loop
                print("Not importing %s into %s for %s" % (self.fname, self.table, prefix))
                continue
            stmt = '''INSERT INTO %s (%s) VALUES (%s)''' % (
                self.table,
                (', '.join([x for x in fields if x[0] != '_'] + self.extra_keys)),
                (', '.join([":" + x for x in fields if x[0] != '_'] + self.extra_values))
            )

            # This does the actual insertions.  Passed the INSERT
            # statement and then an iterator over dictionaries.  Each
            # dictionary is inserted.
            if self.print_progress:
                print('Importing %s into %s for %s' % (self.fname, self.table, prefix))
            # the first row was consumed by fetching the fields
            # (this could be optimized)
            from itertools import chain
            rows = chain([row], self.gen_rows([csv_reader], [prefix]))
            cur.executemany(stmt, rows)
            conn.commit()


Instruction:
Can you generate the documentation for the following Python 3 function
def import_(self, conn):
        if self.print_progress:
            print('Beginning', self.__class__.__name__)
        # what is this mystical self._conn ?
        self._conn = conn

        self.create_table(conn)
        # This does insertions
        if self.mode in ('all', 'import') and self.fname and self.exists() and self.table not in ignore_tables:
            self.insert_data(conn)
        # This makes indexes in the DB.
        if self.mode in ('all', 'index') and hasattr(self, 'index'):
            self.create_index(conn)
        # Any post-processing to be done after the full import.
        if self.mode in ('all', 'import') and hasattr(self, 'post_import'):
            self.run_post_import(conn)
        # Commit it all
        conn.commit()

Response: Copy data and store in the connection object.


Instruction:
How would you explain what the following Python 3 function does
def copy(cls, conn, **where):
        cur = conn.cursor()
        if where and cls.copy_where:
            copy_where = cls.copy_where.format(**where)
            # print(copy_where)
        else:
            copy_where = ''
        cur.execute('INSERT INTO %s '
                    'SELECT * FROM source.%s %s' % (cls.table, cls.table, copy_where))

Response: Copy data from one table to another while filtering data at the same time.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_journey_legs_to_target(self, target, fastest_path=True, min_boardings=False, all_leg_sections=True,
                                                   ignore_walk=False, diff_threshold=None, diff_path=None):
        """
        Returns a dataframe of aggregated sections from source nodes to target. The returned sections are either
        transfer point to transfer point or stop to stop. In a before after setting, the results can be filtered based
        on values in a difference db.
        :param target:
        :param fastest_path:
        :param min_boardings:
        :param all_leg_sections:
        :param ignore_walk:
        :param diff_threshold:
        :param diff_path:
        :return:
        """
        assert not (fastest_path and min_boardings)
        if min_boardings:
            raise NotImplementedError
        if all_leg_sections and diff_threshold:
            raise NotImplementedError

        added_constraints = ""
        add_diff = ""
        if fastest_path:
            added_constraints += " AND journeys.pre_journey_wait_fp>=0"
        if ignore_walk:
            added_constraints += " AND legs.trip_I >= 0"
        if diff_path and diff_threshold:
            self.conn = attach_database(self.conn, diff_path, name="diff")
            add_diff = ", diff.diff_temporal_distance"
            added_constraints += " AND abs(diff_temporal_distance.diff_mean) >= %s " \
                                 "AND diff_temporal_distance.from_stop_I = journeys.from_stop_I " \
                                 "AND diff_temporal_distance.to_stop_I = journeys.to_stop_I" % (diff_threshold,)

        if all_leg_sections:
            df = self._get_journey_legs_to_target_with_all_sections(target, added_constraints)
        else:
            query = """SELECT from_stop_I, to_stop_I, coalesce(type, -1) AS type,
                         count(*) AS n_trips
                         FROM
                         (SELECT legs.* FROM legs, journeys %s
                         WHERE journeys.journey_id = legs.journey_id AND journeys.to_stop_I = %s %s) q1
                         LEFT JOIN (SELECT * FROM other.trips, other.routes WHERE trips.route_I = routes.route_I) q2
                         ON q1.trip_I = q2.trip_I
                         GROUP BY from_stop_I, to_stop_I, type""" % (add_diff, str(target), added_constraints)
            df = read_sql_query(query, self.conn)

        return df

Response: Get all journey legs from source nodes to target.


Instruction:
Given the following Python 3 function, write the documentation
def get_upstream_stops_ratio(self, target, trough_stops, ratio):
        if isinstance(trough_stops, list):
            trough_stops = ",".join(trough_stops)
        query = """SELECT stops.* FROM other.stops,
                    (SELECT q2.from_stop_I AS stop_I FROM
                    (SELECT journeys.from_stop_I, count(*) AS n_total FROM journeys
                    WHERE journeys.to_stop_I = {target}
                    GROUP BY from_stop_I) q1,
                    (SELECT journeys.from_stop_I, count(*) AS n_trough FROM journeys, legs
                    WHERE journeys.journey_id=legs.journey_id AND legs.from_stop_I IN ({trough_stops}) AND journeys.to_stop_I = {target}
                    GROUP BY journeys.from_stop_I) q2
                    WHERE q1.from_stop_I = q2.from_stop_I AND n_trough/(n_total*1.0) >= {ratio}) q1
                    WHERE stops.stop_I = q1.stop_I""".format(target=target, trough_stops=trough_stops, ratio=ratio)
        df = read_sql_query(query, self.conn)
        return df

Response: Returns the set of upstream stops for which the ratio or higher proportion of trips to the target passes trough a set of trough stops.


Instruction:
Can you tell what is the following Python 3 function doing
def get_spatial_bounds(gtfs, as_dict=False):
    stats = get_stats(gtfs)
    lon_min = stats['lon_min']
    lon_max = stats['lon_max']
    lat_min = stats['lat_min']
    lat_max = stats['lat_max']
    if as_dict:
        return {'lon_min': lon_min, 'lon_max': lon_max, 'lat_min': lat_min, 'lat_max': lat_max}
    else:
        return lon_min, lon_max, lat_min, lat_max

Response: Returns the minimum and maximum longitude and latitude of the node in the specified GTFS.


Instruction:
Can you write a function in Python 3 where it
gets median latitude AND longitude of stops in GTFS

Response:
def get_median_lat_lon_of_stops(gtfs):
    """
    Get median latitude AND longitude of stops

    Parameters
    ----------
    gtfs: GTFS

    Returns
    -------
    median_lat : float
    median_lon : float
    """
    stops = gtfs.get_table("stops")
    median_lat = numpy.percentile(stops['lat'].values, 50)
    median_lon = numpy.percentile(stops['lon'].values, 50)
    return median_lat, median_lon


Instruction:
Can you generate the documentation for the following Python 3 function
def get_centroid_of_stops(gtfs):
    stops = gtfs.get_table("stops")
    mean_lat = numpy.mean(stops['lat'].values)
    mean_lon = numpy.mean(stops['lon'].values)
    return mean_lat, mean_lon

Response: Get mean latitude AND longitude of stops in GTFS


Instruction:
Write a Python 3 script for
writing data from get_stats to csv file.

Response:
def write_stats_as_csv(gtfs, path_to_csv, re_write=False):
    """
    Writes data from get_stats to csv file

    Parameters
    ----------
    gtfs: GTFS
    path_to_csv: str
        filepath to the csv file to be generated
    re_write:
        insted of appending, create a new one.
    """
    stats_dict = get_stats(gtfs)
    # check if file exist
    if re_write:
            os.remove(path_to_csv)

    #if not os.path.isfile(path_to_csv):
     #   is_new = True
    #else:
     #   is_new = False

    is_new = True
    mode = 'r' if os.path.exists(path_to_csv) else 'w+'
    with open(path_to_csv, mode) as csvfile:
        for line in csvfile:
           if line:
               is_new = False
           else:
               is_new = True

    with open(path_to_csv, 'a') as csvfile:
        if (sys.version_info > (3, 0)):
            delimiter = u","
        else:
            delimiter = b","
        statswriter = csv.writer(csvfile, delimiter=delimiter)
        # write column names if
        if is_new:
            statswriter.writerow([key for key in sorted(stats_dict.keys())])

        row_to_write = []
        # write stats row sorted by column name
        for key in sorted(stats_dict.keys()):
            row_to_write.append(stats_dict[key])
        statswriter.writerow(row_to_write)


Instruction:
Write a Python 3 function that can
get basic statistics of the GTFS data.

Response:
def get_stats(gtfs):
    """
    Get basic statistics of the GTFS data.

    Parameters
    ----------
    gtfs: GTFS

    Returns
    -------
    stats: dict
        A dictionary of various statistics.
        Keys should be strings, values should be inputtable to a database (int, date, str, ...)
        (but not a list)
    """
    stats = {}
    # Basic table counts
    for table in ['agencies', 'routes', 'stops', 'stop_times', 'trips', 'calendar', 'shapes', 'calendar_dates',
                  'days', 'stop_distances', 'frequencies', 'feed_info', 'transfers']:
        stats["n_" + table] = gtfs.get_row_count(table)

    # Agency names
    agencies = gtfs.get_table("agencies")
    stats["agencies"] = "_".join(agencies['name'].values)

    # Stop lat/lon range
    stops = gtfs.get_table("stops")
    lats = stops['lat'].values
    lons = stops['lon'].values
    percentiles = [0, 10, 50, 90, 100]

    try:
        lat_percentiles = numpy.percentile(lats, percentiles)
    except IndexError:
        lat_percentiles = [None] * 5
    lat_min, lat_10, lat_median, lat_90, lat_max = lat_percentiles
    stats["lat_min"] = lat_min
    stats["lat_10"] = lat_10
    stats["lat_median"] = lat_median
    stats["lat_90"] = lat_90
    stats["lat_max"] = lat_max

    try:
        lon_percentiles = numpy.percentile(lons, percentiles)
    except IndexError:
        lon_percentiles = [None] * 5
    lon_min, lon_10, lon_median, lon_90, lon_max = lon_percentiles
    stats["lon_min"] = lon_min
    stats["lon_10"] = lon_10
    stats["lon_median"] = lon_median
    stats["lon_90"] = lon_90
    stats["lon_max"] = lon_max

    if len(lats) > 0:
        stats["height_km"] = wgs84_distance(lat_min, lon_median, lat_max, lon_median) / 1000.
        stats["width_km"] = wgs84_distance(lon_min, lat_median, lon_max, lat_median) / 1000.
    else:
        stats["height_km"] = None
        stats["width_km"] = None

    first_day_start_ut, last_day_start_ut = gtfs.get_day_start_ut_span()
    stats["start_time_ut"] = first_day_start_ut
    if last_day_start_ut is None:
        stats["end_time_ut"] = None
    else:
        # 28 (instead of 24) comes from the GTFS stANDard
        stats["end_time_ut"] = last_day_start_ut + 28 * 3600

    stats["start_date"] = gtfs.get_min_date()
    stats["end_date"] = gtfs.get_max_date()

    # Maximum activity day
    max_activity_date = gtfs.execute_custom_query(
        'SELECT count(*), date '
        'FROM days '
        'GROUP BY date '
        'ORDER BY count(*) DESC, date '
        'LIMIT 1;').fetchone()
    if max_activity_date:
        stats["max_activity_date"] = max_activity_date[1]
        max_activity_hour = gtfs.get_cursor().execute(
            'SELECT count(*), arr_time_hour FROM day_stop_times '
            'WHERE date=? GROUP BY arr_time_hour '
            'ORDER BY count(*) DESC;', (stats["max_activity_date"],)).fetchone()
        if max_activity_hour:
            stats["max_activity_hour"] = max_activity_hour[1]
        else:
            stats["max_activity_hour"] = None

    # Fleet size estimate: considering each line separately
    if max_activity_date and max_activity_hour:
        fleet_size_estimates = _fleet_size_estimate(gtfs, stats['max_activity_hour'], stats['max_activity_date'])
        stats.update(fleet_size_estimates)

    # Compute simple distributions of various columns that have a finite range of values.
    # Commented lines refer to values that are not imported yet, ?

    stats['routes__type__dist'] = _distribution(gtfs, 'routes', 'type')
    # stats['stop_times__pickup_type__dist'] = _distribution(gtfs, 'stop_times', 'pickup_type')
    # stats['stop_times__drop_off_type__dist'] = _distribution(gtfs, 'stop_times', 'drop_off_type')
    # stats['stop_times__timepoint__dist'] = _distribution(gtfs, 'stop_times', 'timepoint')
    stats['calendar_dates__exception_type__dist'] = _distribution(gtfs, 'calendar_dates', 'exception_type')
    stats['frequencies__exact_times__dist'] = _distribution(gtfs, 'frequencies', 'exact_times')
    stats['transfers__transfer_type__dist'] = _distribution(gtfs, 'transfers', 'transfer_type')
    stats['agencies__lang__dist'] = _distribution(gtfs, 'agencies', 'lang')
    stats['stops__location_type__dist'] = _distribution(gtfs, 'stops', 'location_type')
    # stats['stops__wheelchair_boarding__dist'] = _distribution(gtfs, 'stops', 'wheelchair_boarding')
    # stats['trips__wheelchair_accessible__dist'] = _distribution(gtfs, 'trips', 'wheelchair_accessible')
    # stats['trips__bikes_allowed__dist'] = _distribution(gtfs, 'trips', 'bikes_allowed')
    # stats[''] = _distribution(gtfs, '', '')
    stats = _feed_calendar_span(gtfs, stats)

    return stats


Instruction:
Here you have a function in Python 3, explain what it does
def _distribution(gtfs, table, column):
    cur = gtfs.conn.cursor()
    cur.execute('SELECT {column}, count(*) '
                'FROM {table} GROUP BY {column} '
                'ORDER BY {column}'.format(column=column, table=table))
    return ' '.join('%s:%s' % (t, c) for t, c in cur)

Response: Count occurrences of values AND return it as a string.


Instruction:
Implement a Python 3 function for
calculating the maximum number of vehicles in a single terminal and returns a list of the maximum number of vehicles in a single terminal.

Response:
def _fleet_size_estimate(gtfs, hour, date):
    """
    Calculates fleet size estimates by two separate formula:
     1. Considering all routes separately with no interlining and doing a deficit calculation at every terminal
     2. By looking at the maximum number of vehicles in simultaneous movement

    Parameters
    ----------
    gtfs: GTFS
    hour: int
    date: ?

    Returns
    -------
    results: dict
        a dict with keys:
            fleet_size_route_based
            fleet_size_max_movement

    """
    results = {}

    fleet_size_list = []
    cur = gtfs.conn.cursor()
    rows = cur.execute(
        'SELECT type, max(vehicles) '
        'FROM ('
        'SELECT type, direction_id, sum(vehicles) as vehicles '
        'FROM '
        '('
        'SELECT trips.route_I, trips.direction_id, routes.route_id, name, type, count(*) as vehicles, cycle_time_min '
        'FROM trips, routes, days, '
        '('
        'SELECT first_trip.route_I, first_trip.direction_id, first_trip_start_time, first_trip_end_time, '
        'MIN(start_time_ds) as return_trip_start_time, end_time_ds as return_trip_end_time, '
        '(end_time_ds - first_trip_start_time)/60 as cycle_time_min '
        'FROM '
        'trips, '
        '(SELECT route_I, direction_id, MIN(start_time_ds) as first_trip_start_time, '
        'end_time_ds as first_trip_end_time '
        'FROM trips, days '
        'WHERE trips.trip_I=days.trip_I AND start_time_ds >= ? * 3600 '
        'AND start_time_ds <= (? + 1) * 3600 AND date = ? '
        'GROUP BY route_I, direction_id) first_trip '
        'WHERE first_trip.route_I = trips.route_I '
        'AND first_trip.direction_id != trips.direction_id '
        'AND start_time_ds >= first_trip_end_time '
        'GROUP BY trips.route_I, trips.direction_id'
        ') return_trip '
        'WHERE trips.trip_I=days.trip_I AND trips.route_I= routes.route_I '
        'AND date = ? AND trips.route_I = return_trip.route_I '
        'AND trips.direction_id = return_trip.direction_id '
        'AND start_time_ds >= first_trip_start_time '
        'AND start_time_ds < return_trip_end_time '
        'GROUP BY trips.route_I, trips.direction_id '
        'ORDER BY type, name, vehicles desc'
        ') cycle_times '
        'GROUP BY direction_id, type'
        ') vehicles_type '
        'GROUP BY type;', (hour, hour, date, date))
    for row in rows:
        fleet_size_list.append(str(row[0]) + ':' + str(row[1]))
    results['fleet_size_route_based'] = " ".join(fleet_size_list)

    # Fleet size estimate: maximum number of vehicles in movement
    fleet_size_list = []
    fleet_size_dict = {}
    if hour:
        for minute in range(hour * 3600, (hour + 1) * 3600, 60):
            rows = gtfs.conn.cursor().execute(
                'SELECT type, count(*) '
                'FROM trips, routes, days '
                'WHERE trips.route_I = routes.route_I '
                'AND trips.trip_I=days.trip_I '
                'AND start_time_ds <= ? '
                'AND end_time_ds > ? + 60 '
                'AND date = ? '
                'GROUP BY type;',
                (minute, minute, date))

            for row in rows:
                if fleet_size_dict.get(row[0], 0) < row[1]:
                    fleet_size_dict[row[0]] = row[1]

    for key in fleet_size_dict.keys():
        fleet_size_list.append(str(key) + ':' + str(fleet_size_dict[key]))
    results["fleet_size_max_movement"] = ' '.join(fleet_size_list)
    return results


Instruction:
Can you write a function in Python 3 where it
computes the temporal coverage of each source feed in a given GTFS object.

Response:
def _feed_calendar_span(gtfs, stats):
    """
    Computes the temporal coverage of each source feed

    Parameters
    ----------
    gtfs: gtfspy.GTFS object
    stats: dict
        where to append the stats

    Returns
    -------
    stats: dict
    """
    n_feeds = _n_gtfs_sources(gtfs)[0]
    max_start = None
    min_end = None
    if n_feeds > 1:
        for i in range(n_feeds):
            feed_key = "feed_" + str(i) + "_"
            start_key = feed_key + "calendar_start"
            end_key = feed_key + "calendar_end"
            calendar_span = gtfs.conn.cursor().execute(
                'SELECT min(date), max(date) FROM trips, days '
                'WHERE trips.trip_I = days.trip_I AND trip_id LIKE ?;', (feed_key + '%',)).fetchone()

            stats[start_key] = calendar_span[0]
            stats[end_key] = calendar_span[1]
            if calendar_span[0] is not None and calendar_span[1] is not None:
                if not max_start and not min_end:
                    max_start = calendar_span[0]
                    min_end = calendar_span[1]
                else:
                    if gtfs.get_day_start_ut(calendar_span[0]) > gtfs.get_day_start_ut(max_start):
                        max_start = calendar_span[0]
                    if gtfs.get_day_start_ut(calendar_span[1]) < gtfs.get_day_start_ut(min_end):
                        min_end = calendar_span[1]
        stats["latest_feed_start_date"] = max_start
        stats["earliest_feed_end_date"] = min_end
    else:
        stats["latest_feed_start_date"] = stats["start_date"]
        stats["earliest_feed_end_date"] = stats["end_date"]
    return stats


Instruction:
Can you write a function in Python 3 where it
returns the frequency of all types of routes per day.

Response:
def route_frequencies(gtfs, results_by_mode=False):
    """
    Return the frequency of all types of routes per day.

    Parameters
    -----------
    gtfs: GTFS

    Returns
    -------
    pandas.DataFrame with columns
        route_I, type, frequency
    """
    day = gtfs.get_suitable_date_for_daily_extract()
    query = (
        " SELECT f.route_I, type, frequency FROM routes as r"
        " JOIN"
        " (SELECT route_I, COUNT(route_I) as frequency"
        " FROM"
        " (SELECT date, route_I, trip_I"
        " FROM day_stop_times"
        " WHERE date = '{day}'"
        " GROUP by route_I, trip_I)"
        " GROUP BY route_I) as f"
        " ON f.route_I = r.route_I"
        " ORDER BY frequency DESC".format(day=day))

    return pd.DataFrame(gtfs.execute_custom_query_pandas(query))


Instruction:
Can you create a Python 3 function that
returns all the number of vehicles that pass hourly through a stop in a time frame.

Response:
def hourly_frequencies(gtfs, st, et, route_type):
    """
    Return all the number of vehicles (i.e. busses,trams,etc) that pass hourly through a stop in a time frame.

    Parameters
    ----------
    gtfs: GTFS
    st : int
        start time of the time framein unix time
    et : int
        end time of the time frame in unix time
    route_type: int

    Returns
    -------
    numeric pandas.DataFrame with columns
        stop_I, lat, lon, frequency
    """
    timeframe = et-st
    hours = timeframe/ 3600
    day = gtfs.get_suitable_date_for_daily_extract()
    stops = gtfs.get_stops_for_route_type(route_type).T.drop_duplicates().T
    query = ("SELECT * FROM stops as x"
             " JOIN"
             " (SELECT * , COUNT(*)/{h} as frequency"
             " FROM stop_times, days"
             " WHERE stop_times.trip_I = days.trip_I"
             " AND dep_time_ds > {st}"
             " AND dep_time_ds < {et}"
             " AND date = '{day}'"
             " GROUP BY stop_I) as y"
             " ON y.stop_I = x.stop_I".format(h=hours, st=st, et=et, day=day))
    try:
        trips_frequency = gtfs.execute_custom_query_pandas(query).T.drop_duplicates().T
        df = pd.merge(stops[['stop_I', 'lat', 'lon']], trips_frequency[['stop_I', 'frequency']],
                      on='stop_I', how='inner')
        return df.apply(pd.to_numeric)
    except:
        raise ValueError("Maybe too short time frame!")


Instruction:
Create a Python 3 function to
get the sum of vehicle hours in a particular day by route type.

Response:
def get_vehicle_hours_by_type(gtfs, route_type):
    """
    Return the sum of vehicle hours in a particular day by route type.
    """

    day = gtfs.get_suitable_date_for_daily_extract()
    query = (" SELECT * , SUM(end_time_ds - start_time_ds)/3600 as vehicle_hours_type"
             " FROM"
             " (SELECT * FROM day_trips as q1"
             " INNER JOIN"
             " (SELECT route_I, type FROM routes) as q2"
             " ON q1.route_I = q2.route_I"
             " WHERE type = {route_type}"
             " AND date = '{day}')".format(day=day, route_type=route_type))
    df = gtfs.execute_custom_query_pandas(query)
    return df['vehicle_hours_type'].item()


Instruction:
How would you implement a function in Python 3 that
scans the footpaths originating from stop_id and update the stop label with the arrival time.

Response:
def _scan_footpaths(self, stop_id, walk_departure_time):
        """
        Scan the footpaths originating from stop_id

        Parameters
        ----------
        stop_id: int
        """
        for _, neighbor, data in self._walk_network.edges_iter(nbunch=[stop_id], data=True):
            d_walk = data["d_walk"]
            arrival_time = walk_departure_time + d_walk / self._walk_speed
            self._update_stop_label(neighbor, arrival_time)


Instruction:
Given the following Python 3 function, write the documentation
def timeit(method):
    def timed(*args, **kw):
        time_start = time.time()
        result = method(*args, **kw)
        time_end = time.time()
        print('timeit: %r %2.2f sec (%r, %r) ' % (method.__name__, time_end-time_start, str(args)[:20], kw))
        return result

    return timed

Response: A decorator for printing out the execution time for a function.


Instruction:
Write a Python 3 function for
validating and return the set of warnings that are found in the GTFS feed.

Response:
def validate_and_get_warnings(self):
        """
        Validates/checks a given GTFS feed with respect to a number of different issues.

        The set of warnings that are checked for, can be found in the gtfs_validator.ALL_WARNINGS

        Returns
        -------
        warnings: WarningsContainer
        """
        self.warnings_container.clear()
        self._validate_stops_with_same_stop_time()
        self._validate_speeds_and_trip_times()
        self._validate_stop_spacings()
        self._validate_stop_sequence()
        self._validate_misplaced_stops()
        return self.warnings_container


Instruction:
Explain what the following Python 3 code does
def clean_password(self):
        value = self.cleaned_data.get('password')
        if value not in self.valid_passwords:
            raise forms.ValidationError('Incorrect password.')
        return value

Response: Check that the password is valid.


Instruction:
Here you have a function in Python 3, explain what it does
def clean(self):
        cleaned_data = super(AuthForm, self).clean()
        user = self.get_user()
        if self.staff_only and (not user or not user.is_staff):
            raise forms.ValidationError('Sorry, only staff are allowed.')
        if self.superusers_only and (not user or not user.is_superuser):
            raise forms.ValidationError('Sorry, only superusers are allowed.')
        return cleaned_data

Response: When receiving the filled out form check for valid access.


Instruction:
Can you generate the documentation for the following Python 3 function
def authenticate(self, token_value):
        try:
            backend_path, user_id = token_value.split(':', 1)
        except (ValueError, AttributeError):
            return False
        backend = auth.load_backend(backend_path)
        return bool(backend.get_user(user_id))

Response: Check that the password is valid.


Instruction:
Given the following Python 3 function, write the documentation
def get_lockdown_form(form_path):
    if not form_path:
        raise ImproperlyConfigured('No LOCKDOWN_FORM specified.')
    form_path_list = form_path.split(".")
    new_module = ".".join(form_path_list[:-1])
    attr = form_path_list[-1]
    try:
        mod = import_module(new_module)
    except (ImportError, ValueError):
        raise ImproperlyConfigured('Module configured in LOCKDOWN_FORM (%s) to'
                                   ' contain the form class couldn\'t be '
                                   'found.' % new_module)
    try:
        form = getattr(mod, attr)
    except AttributeError:
        raise ImproperlyConfigured('The module configured in LOCKDOWN_FORM '
                                   ' (%s) doesn\'t define a "%s" form.'
                                   % (new_module, attr))
    return form

Response: Return a lockdown form class for a given string pointing to a lockdown form.


Instruction:
Can you create a Python 3 function that
checks if each request is allowed to access the current resource.

Response:
def process_request(self, request):
        """Check if each request is allowed to access the current resource."""
        try:
            session = request.session
        except AttributeError:
            raise ImproperlyConfigured('django-lockdown requires the Django '
                                       'sessions framework')

        # Don't lock down if django-lockdown is disabled altogether.
        if settings.ENABLED is False:
            return None

        # Don't lock down if the client REMOTE_ADDR matched and is part of the
        # exception list.
        if self.remote_addr_exceptions:
            remote_addr_exceptions = self.remote_addr_exceptions
        else:
            remote_addr_exceptions = settings.REMOTE_ADDR_EXCEPTIONS

        if remote_addr_exceptions:
            # If forwarding proxies are used they must be listed as trusted
            trusted_proxies = self.trusted_proxies or settings.TRUSTED_PROXIES

            remote_addr = request.META.get('REMOTE_ADDR')
            if remote_addr in remote_addr_exceptions:
                return None
            if remote_addr in trusted_proxies:
                # If REMOTE_ADDR is a trusted proxy check x-forwarded-for
                x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
                if x_forwarded_for:
                    remote_addr = x_forwarded_for.split(',')[-1].strip()
                    if remote_addr in remote_addr_exceptions:
                        return None

        # Don't lock down if the URL matches an exception pattern.
        if self.url_exceptions:
            url_exceptions = compile_url_exceptions(self.url_exceptions)
        else:
            url_exceptions = compile_url_exceptions(settings.URL_EXCEPTIONS)
        for pattern in url_exceptions:
            if pattern.search(request.path):
                return None

        # Don't lock down if the URL resolves to a whitelisted view.
        try:
            resolved_path = resolve(request.path)
        except Resolver404:
            pass
        else:
            if resolved_path.func in settings.VIEW_EXCEPTIONS:
                return None

        # Don't lock down if outside of the lockdown dates.
        if self.until_date:
            until_date = self.until_date
        else:
            until_date = settings.UNTIL_DATE

        if self.after_date:
            after_date = self.after_date
        else:
            after_date = settings.AFTER_DATE

        if until_date or after_date:
            locked_date = False
            if until_date and datetime.datetime.now() < until_date:
                locked_date = True
            if after_date and datetime.datetime.now() > after_date:
                locked_date = True
            if not locked_date:
                return None

        form_data = request.POST if request.method == 'POST' else None
        if self.form:
            form_class = self.form
        else:
            form_class = get_lockdown_form(settings.FORM)
        form = form_class(data=form_data, **self.form_kwargs)

        authorized = False
        token = session.get(self.session_key)
        if hasattr(form, 'authenticate'):
            if form.authenticate(token):
                authorized = True
        elif token is True:
            authorized = True

        if authorized and self.logout_key and self.logout_key in request.GET:
            if self.session_key in session:
                del session[self.session_key]
            querystring = request.GET.copy()
            del querystring[self.logout_key]
            return self.redirect(request)

        # Don't lock down if the user is already authorized for previewing.
        if authorized:
            return None

        if form.is_valid():
            if hasattr(form, 'generate_token'):
                token = form.generate_token()
            else:
                token = True
            session[self.session_key] = token
            return self.redirect(request)

        page_data = {'until_date': until_date, 'after_date': after_date}
        if not hasattr(form, 'show_form') or form.show_form():
            page_data['form'] = form

        if self.extra_context:
            page_data.update(self.extra_context)

        return render(request, 'lockdown/form.html', page_data)


Instruction:
Can you implement a function in Python 3 that
infers a single node from a pattern.

Response:
def infer(pattern, base_path=None):
    """https://github.com/frictionlessdata/datapackage-py#infer
    """
    package = Package({}, base_path=base_path)
    descriptor = package.infer(pattern)
    return descriptor


Instruction:
Can you create a Python 3 function that
returns the profile with the received ID as a dict

Response:
def get(self, profile_id):
        '''Returns the profile with the received ID as a dict

        If a local copy of the profile exists, it'll be returned. If not, it'll
        be downloaded from the web. The results are cached, so any subsequent
        calls won't hit the filesystem or the web.

        Args:
            profile_id (str): The ID of the profile you want.

        Raises:
            RegistryError: If there was some problem opening the profile file
                or its format was incorrect.
        '''
        if profile_id not in self._profiles:
            try:
                self._profiles[profile_id] = self._get_profile(profile_id)
            except (ValueError,
                    IOError) as e:
                six.raise_from(RegistryError(e), e)
        return self._profiles[profile_id]


Instruction:
How would you explain what the following Python 3 function does
def _get_profile(self, profile_id):
        '''dict: Return the profile with the received ID as a dict (None if it
        doesn't exist).'''
        profile_metadata = self._registry.get(profile_id)
        if not profile_metadata:
            return

        path = self._get_absolute_path(profile_metadata.get('schema_path'))
        url = profile_metadata.get('schema')
        if path:
            try:
                return self._load_json_file(path)
            except IOError as local_exc:
                if not url:
                    raise local_exc

                try:
                    return self._load_json_url(url)
                except IOError:
                    msg = (
                        'Error loading profile locally at "{path}" '
                        'and remotely at "{url}".'
                    ).format(path=path, url=url)
                    six.raise_from(IOError(msg), local_exc)
        elif url:
            return self._load_json_url(url)

Response: Return the profile with the received ID as a dict.


Instruction:
Write a Python 3 function that can
return the registry as dict with profiles keyed by id.

Response:
def _get_registry(self, registry_path_or_url):
        '''dict: Return the registry as dict with profiles keyed by id.'''
        if registry_path_or_url.startswith('http'):
            profiles = self._load_json_url(registry_path_or_url)
        else:
            profiles = self._load_json_file(registry_path_or_url)
        try:
            registry = {}
            for profile in profiles:
                registry[profile['id']] = profile
            return registry
        except KeyError as e:
            msg = (
                'Registry at "{path}" has no "id" column.'
            ).format(path=registry_path_or_url)
            six.raise_from(ValueError(msg), e)


Instruction:
Can you generate the documentation for the following Python 3 function
def _load_json_url(self, url):
        '''dict: Return the JSON at the local path or URL as a dict.'''
        res = requests.get(url)
        res.raise_for_status()

        return res.json()

Response: Return the JSON at the local path or URL as a dict.


Instruction:
Write a Python 3 script for
getting base path of the descriptor if string or return None.

Response:
def get_descriptor_base_path(descriptor):
    """Get descriptor base path if string or return None.
    """

    # Infer from path/url
    if isinstance(descriptor, six.string_types):
        if os.path.exists(descriptor):
            base_path = os.path.dirname(os.path.abspath(descriptor))
        else:
            # suppose descriptor is a URL
            base_path = os.path.dirname(descriptor)

    # Current dir by default
    else:
        base_path = '.'

    return base_path


Instruction:
Make a summary of the following Python 3 code
def dereference_package_descriptor(descriptor, base_path):
    for resource in descriptor.get('resources', []):
        dereference_resource_descriptor(resource, base_path, descriptor)
    return descriptor

Response: Dereference a data package descriptor.


Instruction:
How would you explain what the following Python 3 function does
def dereference_resource_descriptor(descriptor, base_path, base_descriptor=None):
    PROPERTIES = ['schema', 'dialect']
    if base_descriptor is None:
        base_descriptor = descriptor
    for property in PROPERTIES:
        value = descriptor.get(property)

        # URI -> No
        if not isinstance(value, six.string_types):
            continue

        # URI -> Pointer
        if value.startswith('#'):
            try:
                pointer = jsonpointer.JsonPointer(value[1:])
                descriptor[property] = pointer.resolve(base_descriptor)
            except Exception as error:
                message = 'Not resolved Pointer URI "%s" for resource.%s' % (value, property)
                six.raise_from(
                    exceptions.DataPackageException(message),
                    error
                )

        # URI -> Remote
        elif value.startswith('http'):
            try:
                response = requests.get(value)
                response.raise_for_status()
                descriptor[property] = response.json()
            except Exception as error:
                message = 'Not resolved Remote URI "%s" for resource.%s' % (value, property)
                six.raise_from(
                    exceptions.DataPackageException(message),
                    error
                )

        # URI -> Local
        else:
            if not is_safe_path(value):
                raise exceptions.DataPackageException(
                    'Not safe path in Local URI "%s" '
                    'for resource.%s' % (value, property))
            if not base_path:
                raise exceptions.DataPackageException(
                    'Local URI "%s" requires base path '
                    'for resource.%s' % (value, property))
            fullpath = os.path.join(base_path, value)
            try:
                with io.open(fullpath, encoding='utf-8') as file:
                    descriptor[property] = json.load(file)
            except Exception as error:
                message = 'Not resolved Local URI "%s" for resource.%s' % (value, property)
                six.raise_from(
                    exceptions.DataPackageException(message),
                    error
                )

    return descriptor

Response: Dereference resource descriptor (IN-PLACE FOR NOW).


Instruction:
How would you code a function in Python 3 to
expand the descriptor to include all resources in the data package.

Response:
def expand_package_descriptor(descriptor):
    """Apply defaults to data package descriptor (IN-PLACE FOR NOW).
    """
    descriptor.setdefault('profile', config.DEFAULT_DATA_PACKAGE_PROFILE)
    for resource in descriptor.get('resources', []):
        expand_resource_descriptor(resource)
    return descriptor


Instruction:
Can you generate a brief explanation for the following Python 3 code
def expand_resource_descriptor(descriptor):
    descriptor.setdefault('profile', config.DEFAULT_RESOURCE_PROFILE)
    if descriptor['profile'] == 'tabular-data-resource':

        # Schema
        schema = descriptor.get('schema')
        if schema is not None:
            for field in schema.get('fields', []):
                field.setdefault('type', config.DEFAULT_FIELD_TYPE)
                field.setdefault('format', config.DEFAULT_FIELD_FORMAT)
            schema.setdefault('missingValues', config.DEFAULT_MISSING_VALUES)

        # Dialect
        dialect = descriptor.get('dialect')
        if dialect is not None:
            for key, value in config.DEFAULT_DIALECT.items():
                dialect.setdefault(key, value)

    return descriptor

Response: Expand the resource descriptor to include default values.


Instruction:
Make a summary of the following Python 3 code
def is_safe_path(path):
    contains_windows_var = lambda val: re.match(r'%.+%', val)
    contains_posix_var = lambda val: re.match(r'\$.+', val)

    unsafeness_conditions = [
        os.path.isabs(path),
        ('..%s' % os.path.sep) in path,
        path.startswith('~'),
        os.path.expandvars(path) != path,
        contains_windows_var(path),
        contains_posix_var(path),
    ]

    return not any(unsafeness_conditions)

Response: Check if path is safe and allowed.


Instruction:
Here you have a function in Python 3, explain what it does
def _extract_zip_if_possible(descriptor):
    tempdir = None
    result = descriptor
    try:
        if isinstance(descriptor, six.string_types):
            res = requests.get(descriptor)
            res.raise_for_status()
            result = res.content
    except (IOError,
            ValueError,
            requests.exceptions.RequestException):
        pass
    try:
        the_zip = result
        if isinstance(the_zip, bytes):
            try:
                os.path.isfile(the_zip)
            except (TypeError, ValueError):
                # the_zip contains the zip file contents
                the_zip = io.BytesIO(the_zip)
        if zipfile.is_zipfile(the_zip):
            with zipfile.ZipFile(the_zip, 'r') as z:
                _validate_zip(z)
                descriptor_path = [
                    f for f in z.namelist() if f.endswith('datapackage.json')][0]
                tempdir = tempfile.mkdtemp('-datapackage')
                z.extractall(tempdir)
                result = os.path.join(tempdir, descriptor_path)
        else:
            result = descriptor
    except (TypeError,
            zipfile.BadZipfile):
        pass
    if hasattr(descriptor, 'seek'):
        # Rewind descriptor if it's a file, as we read it for testing if it's
        # a zip file
        descriptor.seek(0)
    return (tempdir, result)

Response: Extract the archive if possible.


Instruction:
Given the following Python 3 function, write the documentation
def _validate_zip(the_zip):
    datapackage_jsons = [f for f in the_zip.namelist() if f.endswith('datapackage.json')]
    if len(datapackage_jsons) != 1:
        msg = 'DataPackage must have only one "datapackage.json" (had {n})'
        raise exceptions.DataPackageException(msg.format(n=len(datapackage_jsons)))

Response: Validate zipped data package


Instruction:
Here you have a function in Python 3, explain what it does
def get_resource(self, name):
        for resource in self.resources:
            if resource.name == name:
                return resource
        return None

Response: Get a resource by its name.


Instruction:
Create a Python 3 function for
adding a resource to the current resource set.

Response:
def add_resource(self, descriptor):
        """https://github.com/frictionlessdata/datapackage-py#package
        """
        self.__current_descriptor.setdefault('resources', [])
        self.__current_descriptor['resources'].append(descriptor)
        self.__build()
        return self.__resources[-1]


Instruction:
Write a Python 3 function for
removing a resource from the resource list.

Response:
def remove_resource(self, name):
        """https://github.com/frictionlessdata/datapackage-py#package
        """
        resource = self.get_resource(name)
        if resource:
            predicat = lambda resource: resource.get('name') != name
            self.__current_descriptor['resources'] = list(filter(
                predicat, self.__current_descriptor['resources']))
            self.__build()
        return resource


Instruction:
How would you explain what the following Python 3 function does
def infer(self, pattern=False):

        # Files
        if pattern:

            # No base path
            if not self.__base_path:
                message = 'Base path is required for pattern infer'
                raise exceptions.DataPackageException(message)

            # Add resources
            options = {'recursive': True} if '**' in pattern else {}
            for path in glob.glob(os.path.join(self.__base_path, pattern), **options):
                self.add_resource({'path': os.path.relpath(path, self.__base_path)})

        # Resources
        for index, resource in enumerate(self.resources):
            descriptor = resource.infer()
            self.__current_descriptor['resources'][index] = descriptor
            self.__build()

        # Profile
        if self.__next_descriptor['profile'] == config.DEFAULT_DATA_PACKAGE_PROFILE:
            if self.resources and all(map(lambda resource: resource.tabular, self.resources)):
                self.__current_descriptor['profile'] = 'tabular-data-package'
                self.__build()

        return self.__current_descriptor

Response: Infer the current state of the data package.


Instruction:
Can you create a Python 3 function that
saves the current state of the object to a file or zip file.

Response:
def save(self, target=None, storage=None, **options):
        """https://github.com/frictionlessdata/datapackage-py#package
        """

        # Save package to storage
        if storage is not None:
            if not isinstance(storage, Storage):
                storage = Storage.connect(storage, **options)
            buckets = []
            schemas = []
            for resource in self.resources:
                if resource.tabular:
                    resource.infer()
                    buckets.append(_slugify_resource_name(resource.name))
                    schemas.append(resource.schema.descriptor)
            schemas = list(map(_slugify_foreign_key, schemas))
            storage.create(buckets, schemas, force=True)
            for bucket in storage.buckets:
                resource = self.resources[buckets.index(bucket)]
                storage.write(bucket, resource.iter())

        # Save descriptor to json
        elif str(target).endswith('.json'):
            mode = 'w'
            encoding = 'utf-8'
            if six.PY2:
                mode = 'wb'
                encoding = None
            helpers.ensure_dir(target)
            with io.open(target, mode=mode, encoding=encoding) as file:
                json.dump(self.__current_descriptor, file, indent=4)

        # Save package to zip
        else:
            try:
                with zipfile.ZipFile(target, 'w') as z:
                    descriptor = json.loads(json.dumps(self.__current_descriptor))
                    for index, resource in enumerate(self.resources):
                        if not resource.name:
                            continue
                        if not resource.local:
                            continue
                        path = os.path.abspath(resource.source)
                        basename = resource.descriptor.get('name')
                        resource_format = resource.descriptor.get('format')
                        if resource_format:
                            basename = '.'.join([basename, resource_format.lower()])
                        path_inside_dp = os.path.join('data', basename)
                        z.write(path, path_inside_dp)
                        descriptor['resources'][index]['path'] = path_inside_dp
                    z.writestr('datapackage.json', json.dumps(descriptor))
            except (IOError, zipfile.BadZipfile, zipfile.LargeZipFile) as exception:
                six.raise_from(exceptions.DataPackageException(exception), exception)

        return True


Instruction:
Given the following Python 3 function, write the documentation
def attributes(self):

        # Deprecate
        warnings.warn(
            'Property "package.attributes" is deprecated.',
            UserWarning)

        # Get attributes
        attributes = set(self.to_dict().keys())
        try:
            attributes.update(self.profile.properties.keys())
        except AttributeError:
            pass

        return tuple(attributes)

Response: Returns a tuple of attributes defined in the schema and the data package.


Instruction:
Write a Python 3 script to
return a tuple of the schema s required attributed.

Response:
def required_attributes(self):
        """tuple: The schema's required attributed.
        """

        # Deprecate
        warnings.warn(
            'Property "package.required_attributes" is deprecated.',
            UserWarning)
        required = ()

        # Get required
        try:
            if self.profile.required is not None:
                required = tuple(self.profile.required)
        except AttributeError:
            pass

        return required


Instruction:
Make a summary of the following Python 3 code
def validate(self):

        # Deprecate
        warnings.warn(
            'Property "package.validate" is deprecated.',
            UserWarning)

        descriptor = self.to_dict()
        self.profile.validate(descriptor)

Response: Validate this Data Package.


Instruction:
Write a Python 3 script to
validate the given descriptor.

Response:
def validate(self, descriptor):
        """https://github.com/frictionlessdata/datapackage-py#schema
        """

        # Collect errors
        errors = []
        for error in self._validator.iter_errors(descriptor):
            if isinstance(error, jsonschema.exceptions.ValidationError):
                message = str(error.message)
                if six.PY2:
                    message = message.replace('u\'', '\'')
                descriptor_path = '/'.join(map(str, error.path))
                profile_path = '/'.join(map(str, error.schema_path))
                error = exceptions.ValidationError(
                    'Descriptor validation error: %s '
                    'at "%s" in descriptor and '
                    'at "%s" in profile'
                    % (message, descriptor_path, profile_path))
            errors.append(error)

        # Raise error
        if errors:
            message = 'There are %s validation errors (see exception.errors)' % len(errors)
            raise exceptions.ValidationError(message, errors=errors)

        return True


Instruction:
Here you have a function in Python 3, explain what it does
def tabular(self):
        if self.__current_descriptor.get('profile') == 'tabular-data-resource':
            return True
        if not self.__strict:
            if self.__current_descriptor.get('format') in config.TABULAR_FORMATS:
                return True
            if self.__source_inspection.get('tabular', False):
                return True
        return False

Response: Return True if the resource is tabular.


Instruction:
Make a summary of the following Python 3 code
def iter(self, relations=False, **options):

        # Error for non tabular
        if not self.tabular:
            message = 'Methods iter/read are not supported for non tabular data'
            raise exceptions.DataPackageException(message)

        # Get relations
        if relations:
            relations = self.__get_relations()

        return self.__get_table().iter(relations=relations, **options)

Response: Get an iterator over the resource.


Instruction:
Can you tell what is the following Python 3 function doing
def raw_iter(self, stream=False):

        # Error for inline
        if self.inline:
            message = 'Methods raw_iter/raw_read are not supported for inline data'
            raise exceptions.DataPackageException(message)

        # Get filelike
        if self.multipart:
            filelike = _MultipartSource(self.source, remote=self.remote)
        elif self.remote:
            if self.__table_options.get('http_session'):
                http_session = self.__table_options['http_session']
            else:
                http_session = requests.Session()
                http_session.headers = config.HTTP_HEADERS
            res = http_session.get(self.source, stream=True)
            filelike = res.raw
        else:
            filelike = io.open(self.source, 'rb')

        return filelike

Response: Get the raw content of the resource in a file.


Instruction:
How would you code a function in Python 3 to
read the raw data from the resource table.

Response:
def raw_read(self):
        """https://github.com/frictionlessdata/datapackage-py#resource
        """
        contents = b''
        with self.raw_iter() as filelike:
            for chunk in filelike:
                contents += chunk
        return contents


Instruction:
Create a Python 3 function to
infer the resource from the source file.

Response:
def infer(self, **options):
        """https://github.com/frictionlessdata/datapackage-py#resource
        """
        descriptor = deepcopy(self.__current_descriptor)

        # Blank -> Stop
        if self.__source_inspection.get('blank'):
            return descriptor

        # Name
        if not descriptor.get('name'):
            descriptor['name'] = self.__source_inspection['name']

        # Only for non inline/storage
        if not self.inline and not self.__storage:

            # Format
            if not descriptor.get('format'):
                descriptor['format'] = self.__source_inspection['format']

            # Mediatype
            if not descriptor.get('mediatype'):
                descriptor['mediatype'] = 'text/%s' % descriptor['format']

            # Encoding
            if not descriptor.get('encoding'):
                contents = b''
                with self.raw_iter(stream=True) as stream:
                    for chunk in stream:
                        contents += chunk
                        if len(contents) > 1000: break
                encoding = cchardet.detect(contents)['encoding']
                if encoding is not None:
                    encoding = encoding.lower()
                    descriptor['encoding'] = 'utf-8' if encoding == 'ascii' else encoding

        # Schema
        if not descriptor.get('schema'):
            if self.tabular:
                descriptor['schema'] = self.__get_table().infer(**options)

        # Profile
        if descriptor.get('profile') == config.DEFAULT_RESOURCE_PROFILE:
            if self.tabular:
                descriptor['profile'] = 'tabular-data-resource'

        # Save descriptor
        self.__current_descriptor = descriptor
        self.__build()

        return descriptor


Instruction:
Can you create a Python 3 function that
commits the current state of the resource.

Response:
def commit(self, strict=None):
        """https://github.com/frictionlessdata/datapackage-py#resource
        """
        if strict is not None:
            self.__strict = strict
        elif self.__current_descriptor == self.__next_descriptor:
            return False
        self.__current_descriptor = deepcopy(self.__next_descriptor)
        self.__table = None
        self.__build()
        return True


Instruction:
Write a Python 3 script for
saving the current resource to a file.

Response:
def save(self, target, storage=None, **options):
        """https://github.com/frictionlessdata/datapackage-py#resource
        """

        # Save resource to storage
        if storage is not None:
            if self.tabular:
                self.infer()
                storage.create(target, self.schema.descriptor, force=True)
                storage.write(target, self.iter())

        # Save descriptor to json
        else:
            mode = 'w'
            encoding = 'utf-8'
            if six.PY2:
                mode = 'wb'
                encoding = None
            helpers.ensure_dir(target)
            with io.open(target, mode=mode, encoding=encoding) as file:
                json.dump(self.__current_descriptor, file, indent=4)