Instruction:
Write a Python 3 function for
parsing a play - by - play string describing a play.

Response:
def parse_play(boxscore_id, details, is_hm):
    """Parse play details from a play-by-play string describing a play.

    Assuming valid input, this function returns structured data in a dictionary
    describing the play. If the play detail string was invalid, this function
    returns None.

    :param boxscore_id: the boxscore ID of the play
    :param details: detail string for the play
    :param is_hm: bool indicating whether the offense is at home
    :param returns: dictionary of play attributes or None if invalid
    :rtype: dictionary or None
    """
    # if input isn't a string, return None
    if not details or not isinstance(details, basestring):
        return None

    bs = sportsref.nba.BoxScore(boxscore_id)
    aw, hm = bs.away(), bs.home()
    season = sportsref.nba.Season(bs.season())
    hm_roster = set(bs.basic_stats().query('is_home == True').player_id.values)

    p = {}
    p['detail'] = details
    p['home'] = hm
    p['away'] = aw
    p['is_home_play'] = is_hm

    # parsing field goal attempts
    shotRE = (r'(?P<shooter>{0}) (?P<is_fgm>makes|misses) '
              '(?P<is_three>2|3)\-pt shot').format(PLAYER_RE)
    distRE = r' (?:from (?P<shot_dist>\d+) ft|at rim)'
    assistRE = r' \(assist by (?P<assister>{0})\)'.format(PLAYER_RE)
    blockRE = r' \(block by (?P<blocker>{0})\)'.format(PLAYER_RE)
    shotRE = r'{0}{1}(?:{2}|{3})?'.format(shotRE, distRE, assistRE, blockRE)
    m = re.match(shotRE, details, re.IGNORECASE)
    if m:
        p['is_fga'] = True
        p.update(m.groupdict())
        p['shot_dist'] = p['shot_dist'] if p['shot_dist'] is not None else 0
        p['shot_dist'] = int(p['shot_dist'])
        p['is_fgm'] = p['is_fgm'] == 'makes'
        p['is_three'] = p['is_three'] == '3'
        p['is_assist'] = pd.notnull(p.get('assister'))
        p['is_block'] = pd.notnull(p.get('blocker'))
        shooter_home = p['shooter'] in hm_roster
        p['off_team'] = hm if shooter_home else aw
        p['def_team'] = aw if shooter_home else hm
        return p

    # parsing jump balls
    jumpRE = ((r'Jump ball: (?P<away_jumper>{0}) vs\. (?P<home_jumper>{0})'
               r'(?: \((?P<gains_poss>{0}) gains possession\))?')
              .format(PLAYER_RE))
    m = re.match(jumpRE, details, re.IGNORECASE)
    if m:
        p['is_jump_ball'] = True
        p.update(m.groupdict())
        return p

    # parsing rebounds
    rebRE = (r'(?P<is_oreb>Offensive|Defensive) rebound'
             r' by (?P<rebounder>{0}|Team)').format(PLAYER_RE)
    m = re.match(rebRE, details, re.I)
    if m:
        p['is_reb'] = True
        p.update(m.groupdict())
        p['is_oreb'] = p['is_oreb'].lower() == 'offensive'
        p['is_dreb'] = not p['is_oreb']
        if p['rebounder'] == 'Team':
            p['reb_team'], other = (hm, aw) if is_hm else (aw, hm)
        else:
            reb_home = p['rebounder'] in hm_roster
            p['reb_team'], other = (hm, aw) if reb_home else (aw, hm)
        p['off_team'] = p['reb_team'] if p['is_oreb'] else other
        p['def_team'] = p['reb_team'] if p['is_dreb'] else other
        return p

    # parsing free throws
    ftRE = (r'(?P<ft_shooter>{}) (?P<is_ftm>makes|misses) '
            r'(?P<is_tech_fta>technical )?(?P<is_flag_fta>flagrant )?'
            r'(?P<is_clearpath_fta>clear path )?free throw'
            r'(?: (?P<fta_num>\d+) of (?P<tot_fta>\d+))?').format(PLAYER_RE)
    m = re.match(ftRE, details, re.I)
    if m:
        p['is_fta'] = True
        p.update(m.groupdict())
        p['is_ftm'] = p['is_ftm'] == 'makes'
        p['is_tech_fta'] = bool(p['is_tech_fta'])
        p['is_flag_fta'] = bool(p['is_flag_fta'])
        p['is_clearpath_fta'] = bool(p['is_clearpath_fta'])
        p['is_pf_fta'] = not p['is_tech_fta']
        if p['tot_fta']:
            p['tot_fta'] = int(p['tot_fta'])
        if p['fta_num']:
            p['fta_num'] = int(p['fta_num'])
        ft_home = p['ft_shooter'] in hm_roster
        p['fta_team'] = hm if ft_home else aw
        if not p['is_tech_fta']:
            p['off_team'] = hm if ft_home else aw
            p['def_team'] = aw if ft_home else hm
        return p

    # parsing substitutions
    subRE = (r'(?P<sub_in>{0}) enters the game for '
             r'(?P<sub_out>{0})').format(PLAYER_RE)
    m = re.match(subRE, details, re.I)
    if m:
        p['is_sub'] = True
        p.update(m.groupdict())
        sub_home = p['sub_in'] in hm_roster or p['sub_out'] in hm_roster
        p['sub_team'] = hm if sub_home else aw
        return p

    # parsing turnovers
    toReasons = (r'(?P<to_type>[^;]+)(?:; steal by '
                 r'(?P<stealer>{0}))?').format(PLAYER_RE)
    toRE = (r'Turnover by (?P<to_by>{}|Team) '
            r'\((?:{})\)').format(PLAYER_RE, toReasons)
    m = re.match(toRE, details, re.I)
    if m:
        p['is_to'] = True
        p.update(m.groupdict())
        p['to_type'] = p['to_type'].lower()
        if p['to_type'] == 'offensive foul':
            return None
        p['is_steal'] = pd.notnull(p['stealer'])
        p['is_travel'] = p['to_type'] == 'traveling'
        p['is_shot_clock_viol'] = p['to_type'] == 'shot clock'
        p['is_oob'] = p['to_type'] == 'step out of bounds'
        p['is_three_sec_viol'] = p['to_type'] == '3 sec'
        p['is_backcourt_viol'] = p['to_type'] == 'back court'
        p['is_off_goaltend'] = p['to_type'] == 'offensive goaltending'
        p['is_double_dribble'] = p['to_type'] == 'dbl dribble'
        p['is_discont_dribble'] = p['to_type'] == 'discontinued dribble'
        p['is_carry'] = p['to_type'] == 'palming'
        if p['to_by'] == 'Team':
            p['off_team'] = hm if is_hm else aw
            p['def_team'] = aw if is_hm else hm
        else:
            to_home = p['to_by'] in hm_roster
            p['off_team'] = hm if to_home else aw
            p['def_team'] = aw if to_home else hm
        return p

    # parsing shooting fouls
    shotFoulRE = (r'Shooting(?P<is_block_foul> block)? foul by (?P<fouler>{0})'
                  r'(?: \(drawn by (?P<drew_foul>{0})\))?').format(PLAYER_RE)
    m = re.match(shotFoulRE, details, re.I)
    if m:
        p['is_pf'] = True
        p['is_shot_foul'] = True
        p.update(m.groupdict())
        p['is_block_foul'] = bool(p['is_block_foul'])
        foul_on_home = p['fouler'] in hm_roster
        p['off_team'] = aw if foul_on_home else hm
        p['def_team'] = hm if foul_on_home else aw
        p['foul_team'] = p['def_team']
        return p

    # parsing offensive fouls
    offFoulRE = (r'Offensive(?P<is_charge> charge)? foul '
                 r'by (?P<to_by>{0})'
                 r'(?: \(drawn by (?P<drew_foul>{0})\))?').format(PLAYER_RE)
    m = re.match(offFoulRE, details, re.I)
    if m:
        p['is_pf'] = True
        p['is_off_foul'] = True
        p['is_to'] = True
        p['to_type'] = 'offensive foul'
        p.update(m.groupdict())
        p['is_charge'] = bool(p['is_charge'])
        p['fouler'] = p['to_by']
        foul_on_home = p['fouler'] in hm_roster
        p['off_team'] = hm if foul_on_home else aw
        p['def_team'] = aw if foul_on_home else hm
        p['foul_team'] = p['off_team']
        return p

    # parsing personal fouls
    foulRE = (r'Personal (?P<is_take_foul>take )?(?P<is_block_foul>block )?'
              r'foul by (?P<fouler>{0})(?: \(drawn by '
              r'(?P<drew_foul>{0})\))?').format(PLAYER_RE)
    m = re.match(foulRE, details, re.I)
    if m:
        p['is_pf'] = True
        p.update(m.groupdict())
        p['is_take_foul'] = bool(p['is_take_foul'])
        p['is_block_foul'] = bool(p['is_block_foul'])
        foul_on_home = p['fouler'] in hm_roster
        p['off_team'] = aw if foul_on_home else hm
        p['def_team'] = hm if foul_on_home else aw
        p['foul_team'] = p['def_team']
        return p

    # TODO: parsing double personal fouls
    # double_foul_re = (r'Double personal foul by (?P<fouler1>{0}) and '
    #                   r'(?P<fouler2>{0})').format(PLAYER_RE)
    # m = re.match(double_Foul_re, details, re.I)
    # if m:
    #     p['is_pf'] = True
    #     p.update(m.groupdict())
    #     p['off_team'] =

    # parsing loose ball fouls
    looseBallRE = (r'Loose ball foul by (?P<fouler>{0})'
                   r'(?: \(drawn by (?P<drew_foul>{0})\))?').format(PLAYER_RE)
    m = re.match(looseBallRE, details, re.I)
    if m:
        p['is_pf'] = True
        p['is_loose_ball_foul'] = True
        p.update(m.groupdict())
        foul_home = p['fouler'] in hm_roster
        p['foul_team'] = hm if foul_home else aw
        return p

    # parsing punching fouls
    # TODO

    # parsing away from play fouls
    awayFromBallRE = ((r'Away from play foul by (?P<fouler>{0})'
                       r'(?: \(drawn by (?P<drew_foul>{0})\))?')
                      .format(PLAYER_RE))
    m = re.match(awayFromBallRE, details, re.I)
    if m:
        p['is_pf'] = True
        p['is_away_from_play_foul'] = True
        p.update(m.groupdict())
        foul_on_home = p['fouler'] in hm_roster
        # TODO: figure out who had the ball based on previous play
        p['foul_team'] = hm if foul_on_home else aw
        return p

    # parsing inbound fouls
    inboundRE = (r'Inbound foul by (?P<fouler>{0})'
                 r'(?: \(drawn by (?P<drew_foul>{0})\))?').format(PLAYER_RE)
    m = re.match(inboundRE, details, re.I)
    if m:
        p['is_pf'] = True
        p['is_inbound_foul'] = True
        p.update(m.groupdict())
        foul_on_home = p['fouler'] in hm_roster
        p['off_team'] = aw if foul_on_home else hm
        p['def_team'] = hm if foul_on_home else aw
        p['foul_team'] = p['def_team']
        return p

    # parsing flagrant fouls
    flagrantRE = (r'Flagrant foul type (?P<flag_type>1|2) by (?P<fouler>{0})'
                  r'(?: \(drawn by (?P<drew_foul>{0})\))?').format(PLAYER_RE)
    m = re.match(flagrantRE, details, re.I)
    if m:
        p['is_pf'] = True
        p['is_flagrant'] = True
        p.update(m.groupdict())
        foul_on_home = p['fouler'] in hm_roster
        p['foul_team'] = hm if foul_on_home else aw
        return p

    # parsing clear path fouls
    clearPathRE = (r'Clear path foul by (?P<fouler>{0})'
                   r'(?: \(drawn by (?P<drew_foul>{0})\))?').format(PLAYER_RE)
    m = re.match(clearPathRE, details, re.I)
    if m:
        p['is_pf'] = True
        p['is_clear_path_foul'] = True
        p.update(m.groupdict())
        foul_on_home = p['fouler'] in hm_roster
        p['off_team'] = aw if foul_on_home else hm
        p['def_team'] = hm if foul_on_home else aw
        p['foul_team'] = p['def_team']
        return p

    # parsing timeouts
    timeoutRE = r'(?P<timeout_team>.*?) (?:full )?timeout'
    m = re.match(timeoutRE, details, re.I)
    if m:
        p['is_timeout'] = True
        p.update(m.groupdict())
        isOfficialTO = p['timeout_team'].lower() == 'official'
        name_to_id = season.team_names_to_ids()
        p['timeout_team'] = (
            'Official' if isOfficialTO else
            name_to_id.get(hm, name_to_id.get(aw, p['timeout_team']))
        )
        return p

    # parsing technical fouls
    techRE = (r'(?P<is_hanging>Hanging )?'
              r'(?P<is_taunting>Taunting )?'
              r'(?P<is_ill_def>Ill def )?'
              r'(?P<is_delay>Delay )?'
              r'(?P<is_unsport>Non unsport )?'
              r'tech(?:nical)? foul by '
              r'(?P<tech_fouler>{0}|Team)').format(PLAYER_RE)
    m = re.match(techRE, details, re.I)
    if m:
        p['is_tech_foul'] = True
        p.update(m.groupdict())
        p['is_hanging'] = bool(p['is_hanging'])
        p['is_taunting'] = bool(p['is_taunting'])
        p['is_ill_def'] = bool(p['is_ill_def'])
        p['is_delay'] = bool(p['is_delay'])
        p['is_unsport'] = bool(p['is_unsport'])
        foul_on_home = p['tech_fouler'] in hm_roster
        p['foul_team'] = hm if foul_on_home else aw
        return p

    # parsing ejections
    ejectRE = r'(?P<ejectee>{0}|Team) ejected from game'.format(PLAYER_RE)
    m = re.match(ejectRE, details, re.I)
    if m:
        p['is_ejection'] = True
        p.update(m.groupdict())
        if p['ejectee'] == 'Team':
            p['ejectee_team'] = hm if is_hm else aw
        else:
            eject_home = p['ejectee'] in hm_roster
            p['ejectee_team'] = hm if eject_home else aw
        return p

    # parsing defensive 3 seconds techs
    def3TechRE = (r'(?:Def 3 sec tech foul|Defensive three seconds)'
                  r' by (?P<tech_fouler>{})').format(PLAYER_RE)
    m = re.match(def3TechRE, details, re.I)
    if m:
        p['is_tech_foul'] = True
        p['is_def_three_secs'] = True
        p.update(m.groupdict())
        foul_on_home = p['tech_fouler'] in hm_roster
        p['off_team'] = aw if foul_on_home else hm
        p['def_team'] = hm if foul_on_home else aw
        p['foul_team'] = p['def_team']
        return p

    # parsing violations
    violRE = (r'Violation by (?P<violator>{0}|Team) '
              r'\((?P<viol_type>.*)\)').format(PLAYER_RE)
    m = re.match(violRE, details, re.I)
    if m:
        p['is_viol'] = True
        p.update(m.groupdict())
        if p['viol_type'] == 'kicked_ball':
            p['is_to'] = True
            p['to_by'] = p['violator']
        if p['violator'] == 'Team':
            p['viol_team'] = hm if is_hm else aw
        else:
            viol_home = p['violator'] in hm_roster
            p['viol_team'] = hm if viol_home else aw
        return p

    p['is_error'] = True
    return p


Instruction:
Write a Python 3 function for
fixing up columns of the passed DataFrame such as casting T/F columns to boolean and filling in NaNs for team and opp.

Response:
def clean_features(df):
    """Fixes up columns of the passed DataFrame, such as casting T/F columns to
    boolean and filling in NaNs for team and opp.

    :param df: DataFrame of play-by-play data.
    :returns: Dataframe with cleaned columns.
    """
    df = pd.DataFrame(df)

    bool_vals = set([True, False, None, np.nan])
    sparse_cols = sparse_lineup_cols(df)
    for col in df:

        # make indicator columns boolean type (and fill in NaNs)
        if set(df[col].unique()[:5]) <= bool_vals:
            df[col] = (df[col] == True)

        # fill NaN's in sparse lineup columns to 0
        elif col in sparse_cols:
            df[col] = df[col].fillna(0)

    # fix free throw columns on technicals
    df.loc[df.is_tech_fta, ['fta_num', 'tot_fta']] = 1

    # fill in NaN's/fix off_team and def_team columns
    df.off_team.fillna(method='bfill', inplace=True)
    df.def_team.fillna(method='bfill', inplace=True)
    df.off_team.fillna(method='ffill', inplace=True)
    df.def_team.fillna(method='ffill', inplace=True)

    return df


Instruction:
Can you write a function in Python 3 where it
returns a list of tuples containing the set of period starters for each quarter in the game.

Response:
def get_period_starters(df):
    """TODO
    """

    def players_from_play(play):
        """Figures out what players are in the game based on the players
        mentioned in a play. Returns away and home players as two sets.

        :param play: A dictionary representing a parsed play.
        :returns: (aw_players, hm_players)
        :rtype: tuple of lists
        """
        # if it's a tech FT from between periods, don't count this play
        if (
            play['clock_time'] == '12:00.0' and
            (play.get('is_tech_foul') or play.get('is_tech_fta'))
        ):
            return [], []

        stats = sportsref.nba.BoxScore(play['boxscore_id']).basic_stats()
        home_grouped = stats.groupby('is_home')
        hm_roster = set(home_grouped.player_id.get_group(True).values)
        aw_roster = set(home_grouped.player_id.get_group(False).values)
        player_keys = [
            'assister', 'away_jumper', 'blocker', 'drew_foul', 'fouler',
            'ft_shooter', 'gains_poss', 'home_jumper', 'rebounder', 'shooter',
            'stealer', 'sub_in', 'sub_out', 'to_by'
        ]
        players = [p for p in play[player_keys] if pd.notnull(p)]

        aw_players = [p for p in players if p in aw_roster]
        hm_players = [p for p in players if p in hm_roster]
        return aw_players, hm_players

    # create a mapping { quarter => (away_starters, home_starters) }
    n_periods = df.quarter.nunique()
    period_starters = [(set(), set()) for _ in range(n_periods)]

    # fill out this mapping quarter by quarter
    for qtr, qtr_grp in df.groupby(df.quarter):
        aw_starters, hm_starters = period_starters[qtr-1]
        exclude = set()
        # loop through sets of plays that happen at the "same time"
        for label, time_grp in qtr_grp.groupby(qtr_grp.secs_elapsed):
            # first, if they sub in and weren't already starters, exclude them
            sub_ins = set(time_grp.sub_in.dropna().values)
            exclude.update(sub_ins - aw_starters - hm_starters)
            # second, figure out new starters from each play at this time
            for i, row in time_grp.iterrows():
                aw_players, hm_players = players_from_play(row)
                # update overall sets for the quarter
                aw_starters.update(aw_players)
                hm_starters.update(hm_players)
            # remove excluded (subbed-in) players
            hm_starters -= exclude
            aw_starters -= exclude
            # check whether we have found all starters
            if len(hm_starters) > 5 or len(aw_starters) > 5:
                import ipdb
                ipdb.set_trace()
            if len(hm_starters) >= 5 and len(aw_starters) >= 5:
                break

        if len(hm_starters) != 5 or len(aw_starters) != 5:
            print('WARNING: wrong number of starters for a team in Q{} of {}'
                  .format(qtr, df.boxscore_id.iloc[0]))

    return period_starters


Instruction:
Given the following Python 3 function, write the documentation
def get_sparse_lineups(df):

    # get the lineup data using get_dense_lineups if necessary
    if (set(ALL_LINEUP_COLS) - set(df.columns)):
        lineup_df = get_dense_lineups(df)
    else:
        lineup_df = df[ALL_LINEUP_COLS]

    # create the sparse representation
    hm_lineups = lineup_df[HM_LINEUP_COLS].values
    aw_lineups = lineup_df[AW_LINEUP_COLS].values
    # +1 for home, -1 for away
    hm_df = pd.DataFrame([
        {'{}_in'.format(player_id): 1 for player_id in lineup}
        for lineup in hm_lineups
    ], dtype=int)
    aw_df = pd.DataFrame([
        {'{}_in'.format(player_id): -1 for player_id in lineup}
        for lineup in aw_lineups
    ], dtype=int)
    sparse_df = pd.concat((hm_df, aw_df), axis=1).fillna(0)
    return sparse_df

Response: Get the sparse representation of the lineups.


Instruction:
How would you explain what the following Python 3 function does
def get_dense_lineups(df):
    # TODO: add this precondition to documentation
    assert df['boxscore_id'].nunique() == 1

    def lineup_dict(aw_lineup, hm_lineup):
        """Returns a dictionary of lineups to be converted to columns.
        Specifically, the columns are 'aw_player1' through 'aw_player5' and
        'hm_player1' through 'hm_player5'.

        :param aw_lineup: The away team's current lineup.
        :param hm_lineup: The home team's current lineup.
        :returns: A dictionary of lineups.
        """
        return {
            '{}_player{}'.format(tm, i+1): player
            for tm, lineup in zip(['aw', 'hm'], [aw_lineup, hm_lineup])
            for i, player in enumerate(lineup)
        }

    def handle_sub(row, aw_lineup, hm_lineup):
        """Modifies the aw_lineup and hm_lineup lists based on the substitution
        that takes place in the given row."""
        assert row['is_sub']
        sub_lineup = hm_lineup if row['sub_team'] == row['home'] else aw_lineup
        try:
            # make the sub
            idx = sub_lineup.index(row['sub_out'])
            sub_lineup[idx] = row['sub_in']
        except ValueError:
            # if the sub was double-entered and it's already been executed...
            if (
                row['sub_in'] in sub_lineup
                and row['sub_out'] not in sub_lineup
            ):
                return aw_lineup, hm_lineup
            # otherwise, let's print and pretend this never happened
            print('ERROR IN SUB IN {}, Q{}, {}: {}'
                  .format(row['boxscore_id'], row['quarter'],
                          row['clock_time'], row['detail']))
            raise
        return aw_lineup, hm_lineup

    per_starters = get_period_starters(df)
    cur_qtr = 0
    aw_lineup, hm_lineup = [], []
    df = df.reset_index(drop=True)
    lineups = [{} for _ in range(df.shape[0])]

    # loop through select plays to determine lineups
    sub_or_per_start = df.is_sub | df.quarter.diff().astype(bool)
    for i, row in df.loc[sub_or_per_start].iterrows():
        if row['quarter'] > cur_qtr:
            # first row in a quarter
            assert row['quarter'] == cur_qtr + 1
            # first, finish up the last quarter's lineups
            if cur_qtr > 0 and not df.loc[i-1, 'is_sub']:
                lineups[i-1] = lineup_dict(aw_lineup, hm_lineup)
            # then, move on to the quarter, and enter the starting lineups
            cur_qtr += 1
            aw_lineup, hm_lineup = map(list, per_starters[cur_qtr-1])
            lineups[i] = lineup_dict(aw_lineup, hm_lineup)
            # if the first play in the quarter is a sub, handle that
            if row['is_sub']:
                aw_lineup, hm_lineup = handle_sub(row, aw_lineup, hm_lineup)
        else:
            # during the quarter
            # update lineups first then change lineups based on subs
            lineups[i] = lineup_dict(aw_lineup, hm_lineup)
            if row['is_sub']:
                aw_lineup, hm_lineup = handle_sub(row, aw_lineup, hm_lineup)

    # create and clean DataFrame
    lineup_df = pd.DataFrame(lineups)
    if lineup_df.iloc[-1].isnull().all():
        lineup_df.iloc[-1] = lineup_dict(aw_lineup, hm_lineup)
    lineup_df = lineup_df.groupby(df.quarter).fillna(method='bfill')

    # fill in NaN's based on minutes played
    bool_mat = lineup_df.isnull()
    mask = bool_mat.any(axis=1)
    if mask.any():
        bs = sportsref.nba.BoxScore(df.boxscore_id[0])
        # first, get the true minutes played from the box score
        stats = sportsref.nba.BoxScore(df.boxscore_id.iloc[0]).basic_stats()
        true_mp = pd.Series(
            stats.query('mp > 0')[['player_id', 'mp']]
            .set_index('player_id').to_dict()['mp']
        ) * 60
        # next, calculate minutes played based on the lineup data
        calc_mp = pd.Series(
            {p: (df.secs_elapsed.diff() *
                 [p in row for row in lineup_df.values]).sum()
             for p in stats.query('mp > 0').player_id.values})
        # finally, figure which players are missing minutes
        diff = true_mp - calc_mp
        players_missing = diff.loc[diff.abs() >= 150]
        hm_roster = bs.basic_stats().query('is_home == True').player_id.values
        missing_df = pd.DataFrame(
            {'secs': players_missing.values,
             'is_home': players_missing.index.isin(hm_roster)},
            index=players_missing.index
        )

        if missing_df.empty:
            # TODO: log this as a warning (or error?)
            print('There are NaNs in the lineup data, but no players were '
                  'found to be missing significant minutes')
        else:
            # import ipdb
            # ipdb.set_trace()
            for is_home, group in missing_df.groupby('is_home'):
                player_id = group.index.item()
                tm_cols = (sportsref.nba.pbp.HM_LINEUP_COLS if is_home else
                           sportsref.nba.pbp.AW_LINEUP_COLS)
                row_mask = lineup_df[tm_cols].isnull().any(axis=1)
                lineup_df.loc[row_mask, tm_cols] = (
                    lineup_df.loc[row_mask, tm_cols].fillna(player_id).values
                )

    return lineup_df

Response: Returns a DataFrame that contains the lineups for each player in the game.


Instruction:
Implement a Python 3 function for
finding all plays in a single page.

Response:
def GamePlayFinder(**kwargs):
    """ Docstring will be filled in by __init__.py """

    querystring = _kwargs_to_qs(**kwargs)
    url = '{}?{}'.format(GPF_URL, querystring)
    # if verbose, print url
    if kwargs.get('verbose', False):
        print(url)
    html = utils.get_html(url)
    doc = pq(html)

    # parse
    table = doc('table#all_plays')
    plays = utils.parse_table(table)

    # parse score column
    if 'score' in plays.columns:
        oScore, dScore = zip(*plays.score.apply(lambda s: s.split('-')))
        plays['teamScore'] = oScore
        plays['oppScore'] = dScore
    # add parsed pbp info
    if 'description' in plays.columns:
        plays = pbp.expand_details(plays, detailCol='description')

    return plays


Instruction:
Can you tell what is the following Python 3 function doing
def _kwargs_to_qs(**kwargs):
    # start with defaults
    inpOptDef = inputs_options_defaults()
    opts = {
        name: dct['value']
        for name, dct in inpOptDef.items()
    }

    # clean up keys and values
    for k, v in kwargs.items():
        # pID, playerID => player_id
        if k.lower() in ('pid', 'playerid'):
            del kwargs[k]
            kwargs['player_id'] = v
        # player_id can accept rel URLs
        if k == 'player_id':
            if v.startswith('/players/'):
                kwargs[k] = utils.rel_url_to_id(v)
        # bool => 'Y'|'N'
        if isinstance(v, bool):
            kwargs[k] = 'Y' if v else 'N'
        # tm, team => team_id
        if k.lower() in ('tm', 'team'):
            del kwargs[k]
            kwargs['team_id'] = v
        # yr_min, yr_max => year_min, year_max
        if k.lower() in ('yr_min', 'yr_max'):
            del kwargs[k]
            if k.lower() == 'yr_min':
                kwargs['year_min'] = int(v)
            else:
                kwargs['year_max'] = int(v)
        # wk_min, wk_max => week_num_min, week_num_max
        if k.lower() in ('wk_min', 'wk_max'):
            del kwargs[k]
            if k.lower() == 'wk_min':
                kwargs['week_num_min'] = int(v)
            else:
                kwargs['week_num_max'] = int(v)
        # yr, year, yrs, years => year_min, year_max
        if k.lower() in ('yr', 'year', 'yrs', 'years'):
            del kwargs[k]
            if isinstance(v, collections.Iterable):
                lst = list(v)
                kwargs['year_min'] = min(lst)
                kwargs['year_max'] = max(lst)
            elif isinstance(v, basestring):
                v = list(map(int, v.split(',')))
                kwargs['year_min'] = min(v)
                kwargs['year_max'] = max(v)
            else:
                kwargs['year_min'] = v
                kwargs['year_max'] = v
        # wk, week, wks, weeks => week_num_min, week_num_max
        if k.lower() in ('wk', 'week', 'wks', 'weeks'):
            del kwargs[k]
            if isinstance(v, collections.Iterable):
                lst = list(v)
                kwargs['week_num_min'] = min(lst)
                kwargs['week_num_max'] = max(lst)
            elif isinstance(v, basestring):
                v = list(map(int, v.split(',')))
                kwargs['week_num_min'] = min(v)
                kwargs['week_num_max'] = max(v)
            else:
                kwargs['week_num_min'] = v
                kwargs['week_num_max'] = v
        # if playoff_round defined, then turn on playoff flag
        if k == 'playoff_round':
            kwargs['game_type'] = 'P'
        if isinstance(v, basestring):
            v = v.split(',')
        if not isinstance(v, collections.Iterable):
            v = [v]

    # reset values to blank for defined kwargs
    for k in kwargs:
        if k in opts:
            opts[k] = []

    # update based on kwargs
    for k, v in kwargs.items():
        # if overwriting a default, overwrite it
        if k in opts:
            # if multiple values separated by commas, split em
            if isinstance(v, basestring):
                v = v.split(',')
            elif not isinstance(v, collections.Iterable):
                v = [v]
            for val in v:
                opts[k].append(val)

    opts['request'] = [1]

    qs = '&'.join('{}={}'.format(name, val)
                  for name, vals in sorted(opts.items()) for val in vals)

    return qs

Response: Converts kwargs given to GPF to a querystring.


Instruction:
Can you tell what is the following Python 3 function doing
def inputs_options_defaults():
    # set time variables
    if os.path.isfile(GPF_CONSTANTS_FILENAME):
        modtime = int(os.path.getmtime(GPF_CONSTANTS_FILENAME))
        curtime = int(time.time())
    # if file found and it's been <= a week
    if (os.path.isfile(GPF_CONSTANTS_FILENAME)
            and curtime - modtime <= 7 * 24 * 60 * 60):

        # just read the dict from the cached file
        with open(GPF_CONSTANTS_FILENAME, 'r') as const_f:
            def_dict = json.load(const_f)

    # otherwise, we must regenerate the dict and rewrite it
    else:

        print('Regenerating GPFConstants file')

        html = utils.get_html(GPF_URL)
        doc = pq(html)

        def_dict = {}
        # start with input elements
        for inp in doc('form#play_finder input[name]'):
            name = inp.attrib['name']
            # add blank dict if not present
            if name not in def_dict:
                def_dict[name] = {
                    'value': set(),
                    'options': set(),
                    'type': inp.type
                }

            val = inp.attrib.get('value', '')
            # handle checkboxes and radio buttons
            if inp.type in ('checkbox', 'radio'):
                # deal with default value
                if 'checked' in inp.attrib:
                    def_dict[name]['value'].add(val)
                # add to options
                def_dict[name]['options'].add(val)
            # handle other types of inputs (only other type is hidden?)
            else:
                def_dict[name]['value'].add(val)

        # for dropdowns (select elements)
        for sel in doc.items('form#play_finder select[name]'):
            name = sel.attr['name']
            # add blank dict if not present
            if name not in def_dict:
                def_dict[name] = {
                    'value': set(),
                    'options': set(),
                    'type': 'select'
                }

            # deal with default value
            defaultOpt = sel('option[selected]')
            if len(defaultOpt):
                defaultOpt = defaultOpt[0]
                def_dict[name]['value'].add(defaultOpt.attrib.get('value', ''))
            else:
                def_dict[name]['value'].add(
                    sel('option')[0].attrib.get('value', '')
                )

            # deal with options
            def_dict[name]['options'] = {
                opt.attrib['value'] for opt in sel('option')
                if opt.attrib.get('value')
            }

        # ignore QB kneels by default
        def_dict['include_kneels']['value'] = ['0']

        def_dict.pop('request', None)
        def_dict.pop('use_favorites', None)

        with open(GPF_CONSTANTS_FILENAME, 'w+') as f:
            for k in def_dict:
                try:
                    def_dict[k]['value'] = sorted(
                        list(def_dict[k]['value']), key=int
                    )
                    def_dict[k]['options'] = sorted(
                        list(def_dict[k]['options']), key=int
                    )
                except:
                    def_dict[k]['value'] = sorted(list(def_dict[k]['value']))
                    def_dict[k]['options'] = sorted(
                        list(def_dict[k]['options'])
                    )
            json.dump(def_dict, f)

    return def_dict

Response: Handles scraping options for play finder form.


Instruction:
Can you generate the documentation for the following Python 3 function
def get(self):
        '''
        Please don't do this in production environments.
        '''
        self.write("Memory Session Object Demo:")
        if "sv" in self.session:
            current_value = self.session["sv"]
            self.write("current sv value is %s, and system will delete this value.<br/>" % self.session["sv"])
            self.session.delete("sv")
            if "sv" not in self.session:
                self.write("current sv value is empty")
        else:
            self.write("Session data not found")

Response: Get the current object from the session.


Instruction:
Create a Python 3 function for
expanding the details column of the given dataframe and returns the resulting DataFrame.

Response:
def expand_details(df, detailCol='detail'):
    """Expands the details column of the given dataframe and returns the
    resulting DataFrame.

    :df: The input DataFrame.
    :detailCol: The detail column name.
    :returns: Returns DataFrame with new columns from pbp parsing.
    """
    df = copy.deepcopy(df)
    df['detail'] = df[detailCol]
    dicts = [sportsref.nfl.pbp.parse_play_details(detail) for detail in df['detail'].values]
    # clean up unmatched details
    cols = {c for d in dicts if d for c in d.keys()}
    blankEntry = {c: np.nan for c in cols}
    newDicts = [d if d else blankEntry for d in dicts]
    # get details DataFrame and merge it with original to create main DataFrame
    details = pd.DataFrame(newDicts)
    df = pd.merge(df, details, left_index=True, right_index=True)
    # add isError column
    errors = [i for i, d in enumerate(dicts) if d is None]
    df['isError'] = False
    df.loc[errors, 'isError'] = True
    # fill in some NaN's necessary for _clean_features
    df.loc[0, 'qtr_time_remain'] = '15:00'
    df.qtr_time_remain.fillna(method='bfill', inplace=True)
    df.qtr_time_remain.fillna(
        pd.Series(np.where(df.quarter == 4, '0:00', '15:00')), inplace=True
    )
    # use _clean_features to clean up and add columns
    new_df = df.apply(_clean_features, axis=1)
    return new_df


Instruction:
Can you write a function in Python 3 where it
parses play - by - play string and returns structured data.

Response:
def parse_play_details(details):
    """Parses play details from play-by-play string and returns structured
    data.

    :details: detail string for play
    :returns: dictionary of play attributes
    """

    # if input isn't a string, return None
    if not isinstance(details, basestring):
        return None

    rushOptRE = r'(?P<rushDir>{})'.format(
        r'|'.join(RUSH_OPTS.keys())
    )
    passOptRE = r'(?P<passLoc>{})'.format(
        r'|'.join(PASS_OPTS.keys())
    )

    playerRE = r"\S{6,8}\d{2}"

    # initialize return dictionary - struct
    struct = {}

    # handle challenges
    # TODO: record the play both before & after an overturned challenge
    challengeRE = re.compile(
        r'.+\. (?P<challenger>.+?) challenged.*? the play was '
        '(?P<callUpheld>upheld|overturned)\.',
        re.IGNORECASE
    )
    match = challengeRE.search(details)
    if match:
        struct['isChallenge'] = True
        struct.update(match.groupdict())
        # if overturned, only record updated play
        if 'overturned' in details:
            overturnedIdx = details.index('overturned.')
            newStart = overturnedIdx + len('overturned.')
            details = details[newStart:].strip()
    else:
        struct['isChallenge'] = False

    # TODO: expand on laterals
    struct['isLateral'] = details.find('lateral') != -1

    # create rushing regex
    rusherRE = r"(?P<rusher>{0})".format(playerRE)
    rushOptRE = r"(?: {})?".format(rushOptRE)
    rushYardsRE = r"(?:(?:(?P<rushYds>\-?\d+) yards?)|(?:no gain))"
    # cases: tackle, fumble, td, penalty
    tackleRE = (r"(?: \(tackle by (?P<tackler1>{0})"
                r"(?: and (?P<tackler2>{0}))?\))?"
                .format(playerRE))
    # currently, plays with multiple fumbles record the original fumbler
    # and the final fumble recoverer
    fumbleRE = (
        r"(?:"
        r"\.? ?(?P<fumbler>{0}) fumbles"
        r"(?: \(forced by (?P<fumbForcer>{0})\))?"
        r"(?:.*, recovered by (?P<fumbRecoverer>{0}) at )?"
        r"(?:, ball out of bounds at )?"
        r"(?:(?P<fumbRecFieldSide>[a-z]+)?\-?(?P<fumbRecYdLine>\-?\d+))?"
        r"(?: and returned for (?P<fumbRetYds>\-?\d*) yards)?"
        r")?"
        .format(playerRE))
    tdSafetyRE = r"(?:(?P<isTD>, touchdown)|(?P<isSafety>, safety))?"
    # TODO: offsetting penalties
    penaltyRE = (r"(?:.*?"
                 r"\. Penalty on (?P<penOn>{0}|): "
                 r"(?P<penalty>[^\(,]+)"
                 r"(?: \((?P<penDeclined>Declined)\)|"
                 r", (?P<penYds>\d*) yards?)"
                 r"(?: \(no play\))?"
                 r")?"
                 .format(playerRE))

    rushREstr = (
        r"{}{}(?: for {}{}{}{}{})?"
    ).format(rusherRE, rushOptRE, rushYardsRE, tackleRE, fumbleRE, tdSafetyRE,
             penaltyRE)
    rushRE = re.compile(rushREstr, re.IGNORECASE)

    # create passing regex
    # TODO: capture "defended by X" for defensive stats
    passerRE = r"(?P<passer>{0})".format(playerRE)
    sackRE = (r"(?:sacked (?:by (?P<sacker1>{0})(?: and (?P<sacker2>{0}))? )?"
              r"for (?P<sackYds>\-?\d+) yards?)"
              .format(playerRE))
    # create throw RE
    completeRE = r"pass (?P<isComplete>(?:in)?complete)"
    passOptRE = r"(?: {})?".format(passOptRE)
    targetedRE = r"(?: (?:to |intended for )?(?P<target>{0}))?".format(
        playerRE)
    passYardsRE = r"(?: for (?:(?P<passYds>\-?\d+) yards?|no gain))"
    intRE = (r'(?: is intercepted by (?P<interceptor>{0}) at '.format(playerRE)
             + r'(?:(?P<intFieldSide>[a-z]*)?\-?(?P<intYdLine>\-?\d*))?'
             + r'(?: and returned for (?P<intRetYds>\-?\d+) yards?\.?)?)?')
    throwRE = r'(?:{}{}{}(?:(?:{}|{}){})?)'.format(
        completeRE, passOptRE, targetedRE, passYardsRE, intRE, tackleRE
    )
    passREstr = (
        r"{} (?:{}|{})(?:{}{}{})?"
    ).format(passerRE, sackRE, throwRE, fumbleRE, tdSafetyRE, penaltyRE)
    passRE = re.compile(passREstr, re.IGNORECASE)

    # create kickoff regex
    koKickerRE = r'(?P<koKicker>{0})'.format(playerRE)
    koYardsRE = (r' kicks (?:off|(?P<isOnside>onside))'
                 r' (?:(?P<koYds>\d+) yards?|no gain)')
    nextREs = []
    nextREs.append(
        (r', (?:returned|recovered) by (?P<koReturner>{0})(?: for '
         r'(?:(?P<koRetYds>\-?\d+) yards?|no gain))?').format(playerRE)
    )
    nextREs.append(
        (r'(?P<isMuffedCatch>, muffed catch by )(?P<muffedBy>{0}),'
         r'(?: recovered by (?P<muffRecoverer>{0}))?').format(playerRE) +
        r'(?: and returned for (?:(?P<muffRetYds>\-?\d+) yards|no gain))?'
    )
    nextREs.append(
        r', recovered by (?P<onsideRecoverer>{0})'.format(playerRE)
    )
    nextREs.append(r'(?P<oob>, out of bounds)')
    nextREs.append(r'(?P<isTouchback>, touchback)')
    # TODO: test the following line to fix a small subset of cases
    # (ex: muff -> oob)
    nextRE = ''.join(r'(?:{})?'.format(nre) for nre in nextREs)
    kickoffREstr = r'{}{}{}{}{}{}{}'.format(
        koKickerRE, koYardsRE, nextRE,
        tackleRE, fumbleRE, tdSafetyRE, penaltyRE
    )
    kickoffRE = re.compile(kickoffREstr, re.IGNORECASE)

    # create timeout regex
    timeoutREstr = r'Timeout #(?P<timeoutNum>\d) by (?P<timeoutTeam>.+)'
    timeoutRE = re.compile(timeoutREstr, re.IGNORECASE)

    # create FG regex
    fgKickerRE = r'(?P<fgKicker>{0})'.format(playerRE)
    fgBaseRE = (r' (?P<fgDist>\d+) yard field goal'
                r' (?P<fgGood>good|no good)')
    fgBlockRE = (
        r'(?:, (?P<isBlocked>blocked) by '
        r'(?P<fgBlocker>{0}))?'.format(playerRE) +
        r'(?:, recovered by (?P<fgBlockRecoverer>{0}))?'.format(playerRE) +
        r'(?: and returned for (?:(?P<fgBlockRetYds>\-?\d+) yards?|no gain))?'
    )
    fgREstr = r'{}{}{}{}{}'.format(fgKickerRE, fgBaseRE,
                                   fgBlockRE, tdSafetyRE, penaltyRE)
    fgRE = re.compile(fgREstr, re.IGNORECASE)

    # create punt regex
    punterRE = r'.*?(?P<punter>{0})'.format(playerRE)
    puntBlockRE = (
        (r' punts, (?P<isBlocked>blocked) by (?P<puntBlocker>{0})'
         r'(?:, recovered by (?P<puntBlockRecoverer>{0})').format(playerRE) +
        r'(?: and returned (?:(?P<puntBlockRetYds>\-?\d+) yards|no gain))?)?'
    )
    puntYdsRE = r' punts (?P<puntYds>\d+) yards?'
    nextREs = []
    nextREs.append(r', (?P<isFairCatch>fair catch) by (?P<fairCatcher>{0})'
                   .format(playerRE))
    nextREs.append(r', (?P<oob>out of bounds)')
    nextREs.append(
        (r'(?P<isMuffedCatch>, muffed catch by )(?P<muffedBy>{0}),'
         r' recovered by (?P<muffRecoverer>{0})').format(playerRE) +
        r' and returned for ' +
        r'(?:(?P<muffRetYds>\d+) yards|no gain)'
    )
    nextREs.append(
        r', returned by (?P<puntReturner>{0}) for '.format(playerRE) +
        r'(?:(?P<puntRetYds>\-?\d+) yards?|no gain)'
    )
    nextRE = r'(?:{})?'.format('|'.join(nextREs))
    puntREstr = r'{}(?:{}|{}){}{}{}{}{}'.format(
        punterRE, puntBlockRE, puntYdsRE, nextRE,
        tackleRE, fumbleRE, tdSafetyRE, penaltyRE
    )
    puntRE = re.compile(puntREstr, re.IGNORECASE)

    # create kneel regex
    kneelREstr = (r'(?P<kneelQB>{0}) kneels for '.format(playerRE) +
                  r'(?:(?P<kneelYds>\-?\d+) yards?|no gain)')
    kneelRE = re.compile(kneelREstr, re.IGNORECASE)

    # create spike regex
    spikeREstr = r'(?P<spikeQB>{0}) spiked the ball'.format(playerRE)
    spikeRE = re.compile(spikeREstr, re.IGNORECASE)

    # create XP regex
    extraPointREstr = (r'(?:(?P<xpKicker>{0}) kicks)? ?extra point '
                       r'(?P<xpGood>good|no good)').format(playerRE)
    extraPointRE = re.compile(extraPointREstr, re.IGNORECASE)

    # create 2pt conversion regex
    twoPointREstr = (
        r'Two Point Attempt: (?P<twoPoint>.*?),?\s+conversion\s+'
        r'(?P<twoPointSuccess>succeeds|fails)'
    )
    twoPointRE = re.compile(twoPointREstr, re.IGNORECASE)

    # create penalty regex
    psPenaltyREstr = (
        r'^Penalty on (?P<penOn>{0}|'.format(playerRE) + r'\w{3}): ' +
        r'(?P<penalty>[^\(,]+)(?: \((?P<penDeclined>Declined)\)|' +
        r', (?P<penYds>\d*) yards?|' +
        r'.*?(?: \(no play\)))')
    psPenaltyRE = re.compile(psPenaltyREstr, re.IGNORECASE)

    # try parsing as a kickoff
    match = kickoffRE.search(details)
    if match:
        # parse as a kickoff
        struct['isKickoff'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a timeout
    match = timeoutRE.search(details)
    if match:
        # parse as timeout
        struct['isTimeout'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a field goal
    match = fgRE.search(details)
    if match:
        # parse as a field goal
        struct['isFieldGoal'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a punt
    match = puntRE.search(details)
    if match:
        # parse as a punt
        struct['isPunt'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a kneel
    match = kneelRE.search(details)
    if match:
        # parse as a kneel
        struct['isKneel'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a spike
    match = spikeRE.search(details)
    if match:
        # parse as a spike
        struct['isSpike'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as an XP
    match = extraPointRE.search(details)
    if match:
        # parse as an XP
        struct['isXP'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a 2-point conversion
    match = twoPointRE.search(details)
    if match:
        # parse as a 2-point conversion
        struct['isTwoPoint'] = True
        struct['twoPointSuccess'] = match.group('twoPointSuccess')
        realPlay = sportsref.nfl.pbp.parse_play_details(
            match.group('twoPoint'))
        if realPlay:
            struct.update(realPlay)
        return struct

    # try parsing as a pass
    match = passRE.search(details)
    if match:
        # parse as a pass
        struct['isPass'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a pre-snap penalty
    match = psPenaltyRE.search(details)
    if match:
        # parse as a pre-snap penalty
        struct['isPresnapPenalty'] = True
        struct.update(match.groupdict())
        return struct

    # try parsing as a run
    match = rushRE.search(details)
    if match:
        # parse as a run
        struct['isRun'] = True
        struct.update(match.groupdict())
        return struct

    return None


Instruction:
Make a summary of the following Python 3 code
def _clean_features(struct):
    struct = dict(struct)
    # First, clean up play type bools
    ptypes = ['isKickoff', 'isTimeout', 'isFieldGoal', 'isPunt', 'isKneel',
              'isSpike', 'isXP', 'isTwoPoint', 'isPresnapPenalty', 'isPass',
              'isRun']
    for pt in ptypes:
        struct[pt] = struct[pt] if pd.notnull(struct.get(pt)) else False
    # Second, clean up other existing variables on a one-off basis
    struct['callUpheld'] = struct.get('callUpheld') == 'upheld'
    struct['fgGood'] = struct.get('fgGood') == 'good'
    struct['isBlocked'] = struct.get('isBlocked') == 'blocked'
    struct['isComplete'] = struct.get('isComplete') == 'complete'
    struct['isFairCatch'] = struct.get('isFairCatch') == 'fair catch'
    struct['isMuffedCatch'] = pd.notnull(struct.get('isMuffedCatch'))
    struct['isNoPlay'] = (
        ' (no play)' in struct['detail'] and
        'penalty enforced in end zone' not in struct['detail']
        if struct.get('detail') else False)
    struct['isOnside'] = struct.get('isOnside') == 'onside'
    struct['isSack'] = pd.notnull(struct.get('sackYds'))
    struct['isSafety'] = (struct.get('isSafety') == ', safety' or
                          (struct.get('detail') and
                           'enforced in end zone, safety' in struct['detail']))
    struct['isTD'] = struct.get('isTD') == ', touchdown'
    struct['isTouchback'] = struct.get('isTouchback') == ', touchback'
    struct['oob'] = pd.notnull(struct.get('oob'))
    struct['passLoc'] = PASS_OPTS.get(struct.get('passLoc'), np.nan)
    if struct['isPass']:
        pyds = struct['passYds']
        struct['passYds'] = pyds if pd.notnull(pyds) else 0
    if pd.notnull(struct['penalty']):
        struct['penalty'] = struct['penalty'].strip()
    struct['penDeclined'] = struct.get('penDeclined') == 'Declined'
    if struct['quarter'] == 'OT':
        struct['quarter'] = 5
    struct['rushDir'] = RUSH_OPTS.get(struct.get('rushDir'), np.nan)
    if struct['isRun']:
        ryds = struct['rushYds']
        struct['rushYds'] = ryds if pd.notnull(ryds) else 0
    year = struct.get('season', np.nan)
    struct['timeoutTeam'] = sportsref.nfl.teams.team_ids(year).get(
        struct.get('timeoutTeam'), np.nan
    )
    struct['twoPointSuccess'] = struct.get('twoPointSuccess') == 'succeeds'
    struct['xpGood'] = struct.get('xpGood') == 'good'

    # Third, ensure types are correct
    bool_vars = [
        'fgGood', 'isBlocked', 'isChallenge', 'isComplete', 'isFairCatch',
        'isFieldGoal', 'isKickoff', 'isKneel', 'isLateral', 'isNoPlay',
        'isPass', 'isPresnapPenalty', 'isPunt', 'isRun', 'isSack', 'isSafety',
        'isSpike', 'isTD', 'isTimeout', 'isTouchback', 'isTwoPoint', 'isXP',
        'isMuffedCatch', 'oob', 'penDeclined', 'twoPointSuccess', 'xpGood'
    ]
    int_vars = [
        'down', 'fgBlockRetYds', 'fgDist', 'fumbRecYdLine', 'fumbRetYds',
        'intRetYds', 'intYdLine', 'koRetYds', 'koYds', 'muffRetYds',
        'pbp_score_aw', 'pbp_score_hm', 'passYds', 'penYds', 'puntBlockRetYds',
        'puntRetYds', 'puntYds', 'quarter', 'rushYds', 'sackYds', 'timeoutNum',
        'ydLine', 'yds_to_go'
    ]
    float_vars = [
        'exp_pts_after', 'exp_pts_before', 'home_wp'
    ]
    string_vars = [
        'challenger', 'detail', 'fairCatcher', 'fgBlockRecoverer',
        'fgBlocker', 'fgKicker', 'fieldSide', 'fumbForcer',
        'fumbRecFieldSide', 'fumbRecoverer', 'fumbler', 'intFieldSide',
        'interceptor', 'kneelQB', 'koKicker', 'koReturner', 'muffRecoverer',
        'muffedBy', 'passLoc', 'passer', 'penOn', 'penalty',
        'puntBlockRecoverer', 'puntBlocker', 'puntReturner', 'punter',
        'qtr_time_remain', 'rushDir', 'rusher', 'sacker1', 'sacker2',
        'spikeQB', 'tackler1', 'tackler2', 'target', 'timeoutTeam',
        'xpKicker'
    ]
    for var in bool_vars:
        struct[var] = struct.get(var) is True
    for var in int_vars:
        try:
            struct[var] = int(struct.get(var))
        except (ValueError, TypeError):
            struct[var] = np.nan
    for var in float_vars:
        try:
            struct[var] = float(struct.get(var))
        except (ValueError, TypeError):
            struct[var] = np.nan
    for var in string_vars:
        if var not in struct or pd.isnull(struct[var]) or var == '':
            struct[var] = np.nan

    # Fourth, create new helper variables based on parsed variables
    # creating fieldSide and ydline from location
    if struct['isXP']:
        struct['fieldSide'] = struct['ydLine'] = np.nan
    else:
        fieldSide, ydline = _loc_to_features(struct.get('location'))
        struct['fieldSide'] = fieldSide
        struct['ydLine'] = ydline
    # creating secsElapsed (in entire game) from qtr_time_remain and quarter
    if pd.notnull(struct.get('qtr_time_remain')):
        qtr = struct['quarter']
        mins, secs = map(int, struct['qtr_time_remain'].split(':'))
        struct['secsElapsed'] = qtr * 900 - mins * 60 - secs
    # creating columns for turnovers
    struct['isInt'] = pd.notnull(struct.get('interceptor'))
    struct['isFumble'] = pd.notnull(struct.get('fumbler'))
    # create column for isPenalty
    struct['isPenalty'] = pd.notnull(struct.get('penalty'))
    # create columns for EPA
    struct['team_epa'] = struct['exp_pts_after'] - struct['exp_pts_before']
    struct['opp_epa'] = struct['exp_pts_before'] - struct['exp_pts_after']
    return pd.Series(struct)

Response: Cleans up the features collected in parse_play_details.


Instruction:
Can you implement a function in Python 3 that
converts a location string to a tuple of those values the second being an int.

Response:
def _loc_to_features(loc):
    """Converts a location string "{Half}, {YardLine}" into a tuple of those
    values, the second being an int.

    :l: The string from the play by play table representing location.
    :returns: A tuple that separates out the values, making them missing
    (np.nan) when necessary.

    """
    if loc:
        if isinstance(loc, basestring):
            loc = loc.strip()
            if ' ' in loc:
                r = loc.split()
                r[0] = r[0].lower()
                r[1] = int(r[1])
            else:
                r = (np.nan, int(loc))
        elif isinstance(loc, float):
            return (np.nan, 50)
    else:
        r = (np.nan, np.nan)
    return r


Instruction:
How would you implement a function in Python 3 that
functions that adds team and opp columns to the DataFrame.

Response:
def _add_team_columns(features):
    """Function that adds 'team' and 'opp' columns to the features by iterating
    through the rows in order. A precondition is that the features dicts are in
    order in a continuous game sense and that all rows are from the same game.

    :features: A DataFrame with each row representing each play (in order).
    :returns: A similar DataFrame but with 'team' and 'opp' columns added.
    """
    features = features.to_dict('records')
    curTm = curOpp = None
    playAfterKickoff = False
    # fill in team and opp columns
    for row in features:
        # if it's a kickoff or the play after a kickoff,
        # figure out who has possession manually
        if row['isKickoff'] or playAfterKickoff:
            curTm, curOpp = _team_and_opp(row)
        else:
            curTm, curOpp = _team_and_opp(row, curTm, curOpp)
        row['team'], row['opp'] = curTm, curOpp
        # set playAfterKickoff
        playAfterKickoff = row['isKickoff']

    features = pd.DataFrame(features)
    features.team.fillna(method='bfill', inplace=True)
    features.opp.fillna(method='bfill', inplace=True)
    # ffill for last row
    features.team.fillna(method='ffill', inplace=True)
    features.opp.fillna(method='ffill', inplace=True)
    return features


Instruction:
Here you have a function in Python 3, explain what it does
def _team_and_opp(struct, curTm=None, curOpp=None):
    # if we don't know the current team, figure it out
    if pd.isnull(curTm):
        if struct['isRun']:
            pID = struct['rusher']
        elif struct['isPass']:
            pID = struct['passer']
        elif struct['isFieldGoal']:
            pID = struct['fgKicker']
        elif struct['isPunt']:
            pID = struct['punter']
        elif struct['isXP']:
            pID = struct['xpKicker']
        elif struct['isKickoff']:
            pID = struct['koKicker']
        elif struct['isSpike']:
            pID = struct['spikeQB']
        elif struct['isKneel']:
            pID = struct['kneelQB']
        else:
            pID = None
        curTm = curOpp = np.nan
        bs = sportsref.nfl.boxscores.BoxScore(struct['boxscore_id'])
        if pID and len(pID) == 3:
            curTm = pID
            curOpp = bs.away() if bs.home() == curTm else bs.home()
        elif pID:
            player = sportsref.nfl.Player(pID)
            gamelog = player.gamelog(kind='B')
            curTm = gamelog.loc[
                gamelog.boxscore_id == struct['boxscore_id'], 'team_id'
            ].item()
            curOpp = bs.home() if bs.home() != curTm else bs.away()

        return curTm, curOpp

    # use row's class to determine when possession changes
    if struct['has_class_divider']:
        return curOpp, curTm
    else:
        return curTm, curOpp

Response: Given a dict representing a play and the current team with the ball and opp returns the team and opp tuples.


Instruction:
Implement a function in Python 3 to
add extra convenience features based on teams with and without availabe possession.

Response:
def _add_team_features(df):
    """Adds extra convenience features based on teams with and without
    possession, with the precondition that the there are 'team' and 'opp'
    specified in row.

    :df: A DataFrame representing a game's play-by-play data after
        _clean_features has been called and 'team' and 'opp' have been added by
        _add_team_columns.
    :returns: A dict with new features in addition to previous features.
    """
    assert df.team.notnull().all()

    homeOnOff = df['team'] == df['home']
    # create column for distToGoal
    df['distToGoal'] = np.where(df['team'] != df['fieldSide'],
                                df['ydLine'], 100 - df['ydLine'])
    df['distToGoal'] = np.where(df['isXP'] | df['isTwoPoint'],
                                2, df['distToGoal'])
    # create column for each team's WP
    df['team_wp'] = np.where(homeOnOff, df['home_wp'], 100. - df['home_wp'])
    df['opp_wp'] = 100. - df['team_wp']
    # create columns for each team's WPA
    df['team_wpa'] = np.where(homeOnOff, df['home_wpa'], -df['home_wpa'])
    df['opp_wpa'] = -df['team_wpa']
    # create column for offense and defense scores if not already there
    assert df['boxscore_id'].nunique() == 1
    bs_id = df['boxscore_id'].values[0]
    bs = sportsref.nfl.boxscores.BoxScore(bs_id)
    df['team_score'] = np.where(df['team'] == bs.home(),
                                df['pbp_score_hm'], df['pbp_score_aw'])
    df['opp_score'] = np.where(df['team'] == bs.home(),
                               df['pbp_score_aw'], df['pbp_score_hm'])

    return df


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_player_stats_table(self, subpage, table_id):
        doc = self.get_sub_doc(subpage)
        table = doc('table#{}'.format(table_id))
        df = sportsref.utils.parse_table(table)
        return df

Response: Helper function for player season stats.


Instruction:
Can you generate the documentation for the following Python 3 function
def initialWinProb(line):
    line = float(line)
    probWin = 1. - norm.cdf(0.5, -line, 13.86)
    probTie = norm.cdf(0.5, -line, 13.86) - norm.cdf(-0.5, -line, 13.86)
    return 100. * (probWin + 0.5 * probTie)

Response: Gets the initial win probability of a game given its Vegas line.


Instruction:
Write a Python 3 function for
getting the career gamelog of the given player.

Response:
def gamelog(self, year=None, kind='R'):
        """Gets the career gamelog of the given player.
        :kind: One of 'R', 'P', or 'B' (for regular season, playoffs, or both).
        Case-insensitive; defaults to 'R'.
        :year: The year for which the gamelog should be returned; if None,
        return entire career gamelog. Defaults to None.
        :returns: A DataFrame with the player's career gamelog.
        """
        url = self._subpage_url('gamelog', None)  # year is filtered later
        doc = pq(sportsref.utils.get_html(url))
        table = (doc('table#stats') if kind == 'R' else
                 doc('table#stats_playoffs'))
        df = sportsref.utils.parse_table(table)
        if year is not None:
            df = df.query('year == @year').reset_index(drop=True)
        return df


Instruction:
How would you implement a function in Python 3 that
gets yearly passing stats for the player.

Response:
def passing(self, kind='R'):
        """Gets yearly passing stats for the player.

        :kind: One of 'R', 'P', or 'B'. Case-insensitive; defaults to 'R'.
        :returns: Pandas DataFrame with passing stats.
        """
        doc = self.get_doc()
        table = (doc('table#passing') if kind == 'R' else
                 doc('table#passing_playoffs'))
        df = sportsref.utils.parse_table(table)
        return df


Instruction:
Can you generate a brief explanation for the following Python 3 code
def rushing_and_receiving(self, kind='R'):
        doc = self.get_doc()
        table = (doc('table#rushing_and_receiving') if kind == 'R'
                 else doc('table#rushing_and_receiving_playoffs'))
        if not table:
            table = (doc('table#receiving_and_rushing') if kind == 'R'
                     else doc('table#receiving_and_rushing_playoffs'))
        df = sportsref.utils.parse_table(table)
        return df

Response: Gets yearly rushing and receiving stats for the player.


Instruction:
Can you create a Python 3 function that
returns a DataFrame of plays for a given year.

Response:
def _plays(self, year, play_type, expand_details):
        """Returns a DataFrame of plays for a given year for a given play type
        (like rushing, receiving, or passing).

        :year: The year for the season.
        :play_type: A type of play for which there are plays (as of this
        writing, either "passing", "rushing", or "receiving")
        :expand_details: Bool for whether PBP should be parsed.
        :returns: A DataFrame of plays, each row is a play. Returns None if
        there were no such plays in that year.
        """
        url = self._subpage_url('{}-plays'.format(play_type), year)
        doc = pq(sportsref.utils.get_html(url))
        table = doc('table#all_plays')
        if table:
            if expand_details:
                plays = sportsref.nfl.pbp.expand_details(
                    sportsref.utils.parse_table(table), detailCol='description'
                )
                return plays
            else:
                return sportsref.utils.parse_table(table)
        else:
            return None


Instruction:
Write a Python 3 function for
returning a DataFrame of advanced splits data for a player - year. Note that the year is only used for the player - year.

Response:
def advanced_splits(self, year=None):
        """Returns a DataFrame of advanced splits data for a player-year. Note:
            only go back to 2012.

        :year: The year for the season in question. If None, returns career
        advanced splits.
        :returns: A DataFrame of advanced splits data.
        """
        # get the table
        url = self._subpage_url('splits', year)
        doc = pq(sportsref.utils.get_html(url))
        table = doc('table#advanced_splits')
        df = sportsref.utils.parse_table(table)
        # cleaning the data
        if not df.empty:
            df.split_type.fillna(method='ffill', inplace=True)
        return df


Instruction:
Explain what the following Python 3 code does
def _simple_year_award(self, award_id):
        doc = self.get_doc()
        table = doc('div#leaderboard_{} table'.format(award_id))
        return list(map(int, sportsref.utils.parse_awards_table(table)))

Response: Template for simple year award functions that simply list years for the award.


Instruction:
Can you create a Python 3 function that
returns a mapping from team ID to full team name for a given season.

Response:
def team_names(year):
    """Returns a mapping from team ID to full team name for a given season.
    Example of a full team name: "New England Patriots"

    :year: The year of the season in question (as an int).
    :returns: A dictionary with teamID keys and full team name values.
    """
    doc = pq(sportsref.utils.get_html(sportsref.nfl.BASE_URL + '/teams/'))
    active_table = doc('table#teams_active')
    active_df = sportsref.utils.parse_table(active_table)
    inactive_table = doc('table#teams_inactive')
    inactive_df = sportsref.utils.parse_table(inactive_table)
    df = pd.concat((active_df, inactive_df))
    df = df.loc[~df['has_class_partial_table']]
    ids = df.team_id.str[:3].values
    names = [tr('th a') for tr in active_table('tr').items()]
    names.extend(tr('th a') for tr in inactive_table('tr').items())
    names = [_f for _f in names if _f]
    names = [lst[0].text_content() for lst in names]
    # combine IDs and team names into pandas series
    series = pd.Series(names, index=ids)
    # create a mask to filter to teams from the given year
    mask = ((df.year_min <= year) & (year <= df.year_max)).values
    # filter, convert to a dict, and return
    return series[mask].to_dict()


Instruction:
Can you tell what is the following Python 3 function doing
def team_ids(year):
    names = team_names(year)
    return {v: k for k, v in names.items()}

Response: Returns a mapping from team name to team ID for a given season. Inverse of team_names mapping of team_names. Example of a full team name : New England Patriots


Instruction:
Can you generate a brief explanation for the following Python 3 code
def name(self):
        doc = self.get_main_doc()
        headerwords = doc('div#meta h1')[0].text_content().split()
        lastIdx = headerwords.index('Franchise')
        teamwords = headerwords[:lastIdx]
        return ' '.join(teamwords)

Response: Returns the real name of the franchise given the team ID.


Instruction:
Write a Python 3 script for
returning the roster table for the given year.

Response:
def roster(self, year):
        """Returns the roster table for the given year.

        :year: The year for which we want the roster; defaults to current year.
        :returns: A DataFrame containing roster information for that year.
        """
        doc = self.get_year_doc('{}_roster'.format(year))
        roster_table = doc('table#games_played_team')
        df = sportsref.utils.parse_table(roster_table)
        starter_table = doc('table#starters')
        if not starter_table.empty:
            start_df = sportsref.utils.parse_table(starter_table)
            start_df = start_df.dropna(axis=0, subset=['position'])
            starters = start_df.set_index('position').player_id
            df['is_starter'] = df.player_id.isin(starters)
            df['starting_pos'] = df.player_id.map(
                lambda pid: (starters[starters == pid].index[0]
                             if pid in starters.values else None)
            )
        return df


Instruction:
Can you implement a function in Python 3 that
gets the list of BoxScore objects corresponding to the box scores from the current log entry for that year.

Response:
def boxscores(self, year):
        """Gets list of BoxScore objects corresponding to the box scores from
        that year.

        :year: The year for which we want the boxscores; defaults to current
        year.
        :returns: np.array of strings representing boxscore IDs.
        """
        doc = self.get_year_doc(year)
        table = doc('table#games')
        df = sportsref.utils.parse_table(table)
        if df.empty:
            return np.array([])
        return df.boxscore_id.values


Instruction:
Write a Python 3 function that can
return a PyQuery object containing the info from the meta div at the given year page with the given keyword.

Response:
def _year_info_pq(self, year, keyword):
        """Returns a PyQuery object containing the info from the meta div at
        the top of the team year page with the given keyword.

        :year: Int representing the season.
        :keyword: A keyword to filter to a single p tag in the meta div.
        :returns: A PyQuery object for the selected p element.
        """
        doc = self.get_year_doc(year)
        p_tags = doc('div#meta div:not(.logo) p')
        texts = [p_tag.text_content().strip() for p_tag in p_tags]
        try:
            return next(
                pq(p_tag) for p_tag, text in zip(p_tags, texts)
                if keyword.lower() in text.lower()
            )
        except StopIteration:
            if len(texts):
                raise ValueError('Keyword not found in any p tag.')
            else:
                raise ValueError('No meta div p tags found.')


Instruction:
Can you write a function in Python 3 where it
returns the head coaches for a given year.

Response:
def head_coaches_by_game(self, year):
        """Returns head coach data by game.

        :year: An int representing the season in question.
        :returns: An array with an entry per game of the season that the team
        played (including playoffs). Each entry is the head coach's ID for that
        game in the season.
        """
        coach_str = self._year_info_pq(year, 'Coach').text()
        regex = r'(\S+?) \((\d+)-(\d+)-(\d+)\)'
        coachAndTenure = []
        m = True
        while m:
            m = re.search(regex, coach_str)
            coachID, wins, losses, ties = m.groups()
            nextIndex = m.end(4) + 1
            coachStr = coachStr[nextIndex:]
            tenure = int(wins) + int(losses) + int(ties)
            coachAndTenure.append((coachID, tenure))

        coachIDs = [
            cID for cID, games in coachAndTenure for _ in range(games)
        ]
        return np.array(coachIDs[::-1])


Instruction:
Create a Python 3 function to
return the number of regular season wins a team in a year.

Response:
def wins(self, year):
        """Returns the # of regular season wins a team in a year.

        :year: The year for the season in question.
        :returns: The number of regular season wins.
        """
        schedule = self.schedule(year)
        if schedule.empty:
            return np.nan
        return schedule.query('week_num <= 17').is_win.sum()


Instruction:
Write a Python 3 function for
returning a Pandas DataFrame with schedule information for the given year.

Response:
def schedule(self, year):
        """Returns a DataFrame with schedule information for the given year.

        :year: The year for the season in question.
        :returns: Pandas DataFrame with schedule information.
        """
        doc = self.get_year_doc(year)
        table = doc('table#games')
        df = sportsref.utils.parse_table(table)
        if df.empty:
            return pd.DataFrame()
        df = df.loc[df['week_num'].notnull()]
        df['week_num'] = np.arange(len(df)) + 1
        df['is_win'] = df['game_outcome'] == 'W'
        df['is_loss'] = df['game_outcome'] == 'L'
        df['is_tie'] = df['game_outcome'] == 'T'
        df['is_bye'] = df['game_outcome'].isnull()
        df['is_ot'] = df['overtime'].notnull()
        return df


Instruction:
Can you generate a brief explanation for the following Python 3 code
def srs(self, year):
        try:
            srs_text = self._year_info_pq(year, 'SRS').text()
        except ValueError:
            return None
        m = re.match(r'SRS\s*?:\s*?(\S+)', srs_text)
        if m:
            return float(m.group(1))
        else:
            return None

Response: Returns the SRS for a team in a year.


Instruction:
Implement a function in Python 3 to
return the SOS of a team in a year based on SRS.

Response:
def sos(self, year):
        """Returns the SOS (Strength of Schedule) for a team in a year, based
        on SRS.

        :year: The year for the season in question.
        :returns: A float of SOS.
        """
        try:
            sos_text = self._year_info_pq(year, 'SOS').text()
        except ValueError:
            return None
        m = re.search(r'SOS\s*:\s*(\S+)', sos_text)
        if m:
            return float(m.group(1))
        else:
            return None


Instruction:
Can you implement a function in Python 3 that
returns the coach ID for the team s OC in a given year.

Response:
def off_coordinator(self, year):
        """Returns the coach ID for the team's OC in a given year.

        :year: An int representing the year.
        :returns: A string containing the coach ID of the OC.
        """
        try:
            oc_anchor = self._year_info_pq(year, 'Offensive Coordinator')('a')
            if oc_anchor:
                return oc_anchor.attr['href']
        except ValueError:
            return None


Instruction:
Create a Python 3 function for
returning the coach ID for the team s DC in a given year.

Response:
def def_coordinator(self, year):
        """Returns the coach ID for the team's DC in a given year.

        :year: An int representing the year.
        :returns: A string containing the coach ID of the DC.
        """
        try:
            dc_anchor = self._year_info_pq(year, 'Defensive Coordinator')('a')
            if dc_anchor:
                return dc_anchor.attr['href']
        except ValueError:
            return None


Instruction:
How would you code a function in Python 3 to
return the ID for the stadium in which the team played in a given year.

Response:
def stadium(self, year):
        """Returns the ID for the stadium in which the team played in a given
        year.

        :year: The year in question.
        :returns: A string representing the stadium ID.
        """
        anchor = self._year_info_pq(year, 'Stadium')('a')
        return sportsref.utils.rel_url_to_id(anchor.attr['href'])


Instruction:
Write a Python 3 script to
return the name of the offensive scheme the team ran in the given year.

Response:
def off_scheme(self, year):
        """Returns the name of the offensive scheme the team ran in the given
        year.

        :year: Int representing the season year.
        :returns: A string representing the offensive scheme.
        """
        scheme_text = self._year_info_pq(year, 'Offensive Scheme').text()
        m = re.search(r'Offensive Scheme[:\s]*(.+)\s*', scheme_text, re.I)
        if m:
            return m.group(1)
        else:
            return None


Instruction:
Here you have a function in Python 3, explain what it does
def def_alignment(self, year):
        scheme_text = self._year_info_pq(year, 'Defensive Alignment').text()
        m = re.search(r'Defensive Alignment[:\s]*(.+)\s*', scheme_text, re.I)
        if m:
            return m.group(1)
        else:
            return None

Response:
Returns the name of the defensive alignment the team ran in the
        given year.


Instruction:
How would you code a function in Python 3 to
return a Series of team stats from the team - season page.

Response:
def team_stats(self, year):
        """Returns a Series (dict-like) of team stats from the team-season
        page.

        :year: Int representing the season.
        :returns: A Series of team stats.
        """
        doc = self.get_year_doc(year)
        table = doc('table#team_stats')
        df = sportsref.utils.parse_table(table)
        if df.empty:
            return pd.Series()
        return df.loc[df.player_id == 'Team Stats'].iloc[0]


Instruction:
Write a Python 3 script for
returning a Series of team s opponent s stats from the team - season page.

Response:
def opp_stats(self, year):
        """Returns a Series (dict-like) of the team's opponent's stats from the
        team-season page.

        :year: Int representing the season.
        :returns: A Series of team stats.
        """
        doc = self.get_year_doc(year)
        table = doc('table#team_stats')
        df = sportsref.utils.parse_table(table)
        return df.loc[df.player_id == 'Opp. Stats'].iloc[0]


Instruction:
Can you create a Python 3 function that
returns a Pandas DataFrame of offensive team splits for a season.

Response:
def off_splits(self, year):
        """Returns a DataFrame of offensive team splits for a season.

        :year: int representing the season.
        :returns: Pandas DataFrame of split data.
        """
        doc = self.get_year_doc('{}_splits'.format(year))
        tables = doc('table.stats_table')
        dfs = [sportsref.utils.parse_table(table) for table in tables.items()]
        dfs = [
            df.assign(split=df.columns[0])
            .rename(columns={df.columns[0]: 'split_value'})
            for df in dfs
        ]
        if not dfs:
            return pd.DataFrame()
        return pd.concat(dfs).reset_index(drop=True)


Instruction:
Can you implement a function in Python 3 that
gets the HTML for the given URL using a GET request.

Response:
def get_html(url):
    """Gets the HTML for the given URL using a GET request.

    :url: the absolute URL of the desired page.
    :returns: a string of HTML.
    """
    global last_request_time
    with throttle_process_lock:
        with throttle_thread_lock:
            # sleep until THROTTLE_DELAY secs have passed since last request
            wait_left = THROTTLE_DELAY - (time.time() - last_request_time.value)
            if wait_left > 0:
                time.sleep(wait_left)

            # make request
            response = requests.get(url)

            # update last request time for throttling
            last_request_time.value = time.time()

    # raise ValueError on 4xx status code, get rid of comments, and return
    if 400 <= response.status_code < 500:
        raise ValueError(
            'Status Code {} received fetching URL "{}"'
            .format(response.status_code, url)
        )
    html = response.text
    html = html.replace('<!--', '').replace('-->', '')

    return html


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_table(table, flatten=True, footer=False):
    if not len(table):
        return pd.DataFrame()

    # get columns
    columns = [c.attrib['data-stat']
               for c in table('thead tr:not([class]) th[data-stat]')]

    # get data
    rows = list(table('tbody tr' if not footer else 'tfoot tr')
                .not_('.thead, .stat_total, .stat_average').items())
    data = [
        [flatten_links(td) if flatten else td.text()
         for td in row.items('th,td')]
        for row in rows
    ]

    # make DataFrame
    df = pd.DataFrame(data, columns=columns, dtype='float')

    # add has_class columns
    allClasses = set(
        cls
        for row in rows
        if row.attr['class']
        for cls in row.attr['class'].split()
    )
    for cls in allClasses:
        df['has_class_' + cls] = [
            bool(row.attr['class'] and
                 cls in row.attr['class'].split())
            for row in rows
        ]

    # cleaning the DataFrame

    df.drop(['ranker', 'Xxx', 'Yyy', 'Zzz'],
            axis=1, inplace=True, errors='ignore')

    # year_id -> year (as int)
    if 'year_id' in df.columns:
        df.rename(columns={'year_id': 'year'}, inplace=True)
        if flatten:
            df.year = df.year.fillna(method='ffill')
            df['year'] = df.year.map(lambda s: str(s)[:4]).astype(int)

    # pos -> position
    if 'pos' in df.columns:
        df.rename(columns={'pos': 'position'}, inplace=True)

    # boxscore_word, game_date -> boxscore_id and separate into Y, M, D columns
    for bs_id_col in ('boxscore_word', 'game_date', 'box_score_text'):
        if bs_id_col in df.columns:
            df.rename(columns={bs_id_col: 'boxscore_id'}, inplace=True)
            break

    # ignore *, +, and other characters used to note things
    df.replace(re.compile(r'[\*\+\u2605]', re.U), '', inplace=True)
    for col in df.columns:
        if hasattr(df[col], 'str'):
            df[col] = df[col].str.strip()

    # player -> player_id and/or player_name
    if 'player' in df.columns:
        if flatten:
            df.rename(columns={'player': 'player_id'}, inplace=True)
            # when flattening, keep a column for names
            player_names = parse_table(table, flatten=False)['player_name']
            df['player_name'] = player_names
        else:
            df.rename(columns={'player': 'player_name'}, inplace=True)

    # team, team_name -> team_id
    for team_col in ('team', 'team_name'):
        if team_col in df.columns:
            # first, get rid of faulty rows
            df = df.loc[~df[team_col].isin(['XXX'])]
            if flatten:
                df.rename(columns={team_col: 'team_id'}, inplace=True)

    # season -> int
    if 'season' in df.columns and flatten:
        df['season'] = df['season'].astype(int)

    # handle date_game columns (different types)
    if 'date_game' in df.columns and flatten:
        date_re = r'month=(?P<month>\d+)&day=(?P<day>\d+)&year=(?P<year>\d+)'
        date_df = df['date_game'].str.extract(date_re, expand=True)
        if date_df.notnull().all(axis=1).any():
            df = pd.concat((df, date_df), axis=1)
        else:
            df.rename(columns={'date_game': 'boxscore_id'}, inplace=True)

    # game_location -> is_home
    if 'game_location' in df.columns and flatten:
        df['game_location'] = df['game_location'].isnull()
        df.rename(columns={'game_location': 'is_home'}, inplace=True)

    # mp: (min:sec) -> float(min + sec / 60), notes -> NaN, new column
    if 'mp' in df.columns and df.dtypes['mp'] == object and flatten:
        mp_df = df['mp'].str.extract(
            r'(?P<m>\d+):(?P<s>\d+)', expand=True).astype(float)
        no_match = mp_df.isnull().all(axis=1)
        if no_match.any():
            df.loc[no_match, 'note'] = df.loc[no_match, 'mp']
        df['mp'] = mp_df['m'] + mp_df['s'] / 60

    # converts number-y things to floats
    def convert_to_float(val):
        # percentages: (number%) -> float(number * 0.01)
        m = re.search(r'([-\.\d]+)\%',
                      val if isinstance(val, basestring) else str(val), re.U)
        try:
            if m:
                return float(m.group(1)) / 100 if m else val
            if m:
                return int(m.group(1)) + int(m.group(2)) / 60
        except ValueError:
            return val
        # salaries: $ABC,DEF,GHI -> float(ABCDEFGHI)
        m = re.search(r'\$[\d,]+',
                      val if isinstance(val, basestring) else str(val), re.U)
        try:
            if m:
                return float(re.sub(r'\$|,', '', val))
        except Exception:
            return val
        # generally try to coerce to float, unless it's an int or bool
        try:
            if isinstance(val, (int, bool)):
                return val
            else:
                return float(val)
        except Exception:
            return val

    if flatten:
        df = df.applymap(convert_to_float)

    df = df.loc[df.astype(bool).any(axis=1)]

    return df

Response: Parses a table from sports - reference sites into a pandas dataframe.


Instruction:
Write a Python 3 function for
parsing an info table like the Game Info table or the Officials table on the PFR Boxscore page.

Response:
def parse_info_table(table):
    """Parses an info table, like the "Game Info" table or the "Officials"
    table on the PFR Boxscore page. Keys are lower case and have spaces/special
    characters converted to underscores.

    :table: PyQuery object representing the HTML table.
    :returns: A dictionary representing the information.
    """
    ret = {}
    for tr in list(table('tr').not_('.thead').items()):
        th, td = list(tr('th, td').items())
        key = th.text().lower()
        key = re.sub(r'\W', '_', key)
        val = sportsref.utils.flatten_links(td)
        ret[key] = val
    return ret


Instruction:
Write a Python 3 function for
flattening relative URLs within text of a table cell to IDs and returns the result.

Response:
def flatten_links(td, _recurse=False):
    """Flattens relative URLs within text of a table cell to IDs and returns
    the result.

    :td: the PyQuery object for the HTML to convert
    :returns: the string with the links flattened to IDs
    """

    # helper function to flatten individual strings/links
    def _flatten_node(c):
        if isinstance(c, basestring):
            return c.strip()
        elif 'href' in c.attrib:
            c_id = rel_url_to_id(c.attrib['href'])
            return c_id if c_id else c.text_content().strip()
        else:
            return flatten_links(pq(c), _recurse=True)

    # if there's no text, just return None
    if td is None or not td.text():
        return '' if _recurse else None

    td.remove('span.note')
    return ''.join(_flatten_node(c) for c in td.contents())


Instruction:
How would you explain what the following Python 3 function does
def rel_url_to_id(url):
    yearRegex = r'.*/years/(\d{4}).*|.*/gamelog/(\d{4}).*'
    playerRegex = r'.*/players/(?:\w/)?(.+?)(?:/|\.html?)'
    boxscoresRegex = r'.*/boxscores/(.+?)\.html?'
    teamRegex = r'.*/teams/(\w{3})/.*'
    coachRegex = r'.*/coaches/(.+?)\.html?'
    stadiumRegex = r'.*/stadiums/(.+?)\.html?'
    refRegex = r'.*/officials/(.+?r)\.html?'
    collegeRegex = r'.*/schools/(\S+?)/.*|.*college=([^&]+)'
    hsRegex = r'.*/schools/high_schools\.cgi\?id=([^\&]{8})'
    bsDateRegex = r'.*/boxscores/index\.f?cgi\?(month=\d+&day=\d+&year=\d+)'
    leagueRegex = r'.*/leagues/(.*_\d{4}).*'
    awardRegex = r'.*/awards/(.+)\.htm'

    regexes = [
        yearRegex,
        playerRegex,
        boxscoresRegex,
        teamRegex,
        coachRegex,
        stadiumRegex,
        refRegex,
        collegeRegex,
        hsRegex,
        bsDateRegex,
        leagueRegex,
        awardRegex,
    ]

    for regex in regexes:
        match = re.match(regex, url, re.I)
        if match:
            return [_f for _f in match.groups() if _f][0]

    # things we don't want to match but don't want to print a WARNING
    if any(
        url.startswith(s) for s in
        (
            '/play-index/',
        )
    ):
        return url

    print('WARNING. NO MATCH WAS FOUND FOR "{}"'.format(url))
    return url

Response: Converts a relative URL to a unique ID.


Instruction:
Given the following Python 3 function, write the documentation
def PlayerSeasonFinder(**kwargs):

    if 'offset' not in kwargs:
        kwargs['offset'] = 0

    playerSeasons = []
    while True:
        querystring = _kwargs_to_qs(**kwargs)
        url = '{}?{}'.format(PSF_URL, querystring)
        if kwargs.get('verbose', False):
            print(url)
        html = utils.get_html(url)
        doc = pq(html)
        table = doc('table#results')
        df = utils.parse_table(table)
        if df.empty:
            break

        thisSeason = list(zip(df.player_id, df.year))
        playerSeasons.extend(thisSeason)

        if doc('*:contains("Next Page")'):
            kwargs['offset'] += 100
        else:
            break

    return playerSeasons

Response: This function returns a list of all the seasons in the current language.


Instruction:
Implement a function in Python 3 to
convert kwargs given to PSF to a querystring.

Response:
def _kwargs_to_qs(**kwargs):
    """Converts kwargs given to PSF to a querystring.

    :returns: the querystring.
    """
    # start with defaults
    inpOptDef = inputs_options_defaults()
    opts = {
        name: dct['value']
        for name, dct in inpOptDef.items()
    }

    # clean up keys and values
    for k, v in kwargs.items():
        del kwargs[k]
        # bool => 'Y'|'N'
        if isinstance(v, bool):
            kwargs[k] = 'Y' if v else 'N'
        # tm, team => team_id
        elif k.lower() in ('tm', 'team'):
            kwargs['team_id'] = v
        # yr, year, yrs, years => year_min, year_max
        elif k.lower() in ('yr', 'year', 'yrs', 'years'):
            if isinstance(v, collections.Iterable):
                lst = list(v)
                kwargs['year_min'] = min(lst)
                kwargs['year_max'] = max(lst)
            elif isinstance(v, basestring):
                v = list(map(int, v.split(',')))
                kwargs['year_min'] = min(v)
                kwargs['year_max'] = max(v)
            else:
                kwargs['year_min'] = v
                kwargs['year_max'] = v
        # pos, position, positions => pos[]
        elif k.lower() in ('pos', 'position', 'positions'):
            if isinstance(v, basestring):
                v = v.split(',')
            elif not isinstance(v, collections.Iterable):
                v = [v]
            kwargs['pos[]'] = v
        # draft_pos, ... => draft_pos[]
        elif k.lower() in (
            'draft_pos', 'draftpos', 'draftposition', 'draftpositions',
            'draft_position', 'draft_positions'
        ):
            if isinstance(v, basestring):
                v = v.split(',')
            elif not isinstance(v, collections.Iterable):
                v = [v]
            kwargs['draft_pos[]'] = v
        # if not one of these cases, put it back in kwargs
        else:
            kwargs[k] = v

    # update based on kwargs
    for k, v in kwargs.items():
        # if overwriting a default, overwrite it (with a list so the
        # opts -> querystring list comp works)
        if k in opts or k in ('pos[]', 'draft_pos[]'):
            # if multiple values separated by commas, split em
            if isinstance(v, basestring):
                v = v.split(',')
            # otherwise, make sure it's a list
            elif not isinstance(v, collections.Iterable):
                v = [v]
            # then, add list of values to the querystring dict *opts*
            opts[k] = v
        if 'draft' in k:
            opts['draft'] = [1]

    opts['request'] = [1]
    opts['offset'] = [kwargs.get('offset', 0)]

    qs = '&'.join(
        '{}={}'.format(urllib.parse.quote_plus(name), val)
        for name, vals in sorted(opts.items()) for val in vals
    )

    return qs


Instruction:
How would you code a function in Python 3 to
read a node from the streamer.

Response:
def _Streamer__read_process(self, path, read_size, cbuf, stop, barrier, cyclic, offset, read_skip, sync):
    """
    Main function for the processes that read from the HDF5 file.

    :param self: A reference to the streamer object that created these processes.
    :param path: The HDF5 path to the node to be read from.
    :param read_size: The length of the block along the outer dimension to read.
    :param cbuf: The circular buffer to place read elements into.
    :param stop: The Event that signals the process to stop reading.
    :param barrier: The Barrier that synchonises read cycles.
    :param cyclic: True if the process should read cyclically.
    :param offset: Offset into the dataset that this process should start reading at.
    :param read_skip: How many element to skip on each iteration.
    :param sync: GuardSynchonizer to order writes to the buffer.
    :return: Nothing
    """
    # Multi-process access to HDF5 seems to behave better there are no top level imports of PyTables.
    import tables as tb
    h5_file = tb.open_file(self.filename, 'r', **self.h5_kw_args)
    ary = h5_file.get_node(path)

    i = offset
    while not stop.is_set():
        vals = ary[i:i + read_size]
        # If the read goes off the end of the dataset, then wrap to the start.
        if i + read_size > len(ary):
            vals = np.concatenate([vals, ary[0:read_size - len(vals)]])

        if sync is None:
            # If no ordering is requested, then just write to the next available space in the buffer.
            with cbuf.put_direct() as put_ary:
                put_ary[:] = vals
        else:
            # Otherwise, use the sync object to ensure that writes occur in the order provided by i.
            # So i = 0 will write first, then i = block_size, then i = 2*block_size, etc...
            # The sync object has two ordered barriers so that acquisition and release of the buffer spaces
            # are synchronized in order, but the actual writing to the buffer can happen simultaneously.
            # If only one barrier were used, writing to the buffer would be linearised.
            with sync.do(cbuf.put_direct(), i, (i+read_size) % len(ary)) as put_ary:
                put_ary[:] = vals

        i += read_skip
        if cyclic:
            # If the next iteration is past the end of the dataset, wrap it around.
            if i >= len(ary):
                i %= len(ary)
                barrier.wait()
        else:
            # But if cyclic mode is disabled, break the loop as the work is now done.
            if i + read_size > len(ary):
                break


Instruction:
How would you code a function in Python 3 to
wait until all processes have reached the barrier.

Response:
def wait(self):
        """Wait until all processes have reached the barrier."""
        with self.cvar:
            self.count.value += 1
            self.cvar.notify_all()
            while self.count.value < self.n_procs:
                self.cvar.wait()


Instruction:
How would you code a function in Python 3 to
wait until all processes have reached the barrier.

Response:
def wait(self):
        """Wait until all processes have reached the barrier."""
        self.barrier_A.wait()
        # The current barrier (barrier_A) is switched with the reserve barrier.
        # This is because the current barrier cannot be safely reset until the reserve barrier has been passed.
        self.barrier_A, self.barrier_B = self.barrier_B, self.barrier_A
        self.barrier_A.reset()


Instruction:
Here you have a function in Python 3, explain what it does
def wait(self, index, next_index=None):
        return OrderedBarrier.Guard(self, index, index+1 if next_index is None else next_index)

Response: Block until the next_index is reached.


Instruction:
Here you have a function in Python 3, explain what it does
def do(self, guard, index, next_index):
        return GuardSynchronizer.Guard(self, guard, index, next_index)

Response: Creates a new guard that requires the resource guard to be entered and exited based on the order provided by index.


Instruction:
Can you generate the documentation for the following Python 3 function
def put(self, v):
        if v is QueueClosed:
            v = -2
        else:
            assert(v >= 0)
        with self.cvar:
            assert(self.size.value < len(self.vals))

            head = (self.tail.value + self.size.value) % len(self.vals)
            self.vals[head] = v
            self.size.value += 1
            self.cvar.notify()

Response: Put an unsigned integer into the queue.


Instruction:
Make a summary of the following Python 3 code
def get(self):
        with self.cvar:
            while True:
                if self.size.value > 0:
                    rval = self.vals[self.tail.value]
                    self.tail.value = (self.tail.value + 1) % len(self.vals)
                    self.size.value -= 1
                    if rval == -2:
                        return QueueClosed
                    assert(rval >= 0)
                    return rval
                self.cvar.wait()

Response: Fetch the next item in the queue. Blocks until an item is available.


Instruction:
Write a Python 3 function for
returning a guard object that allows direct access to the buffer element.

Response:
def put_direct(self):
        """
        Allows direct access to the buffer element.
        Blocks until there is room to write into the buffer.

        :return: A guard object that returns the buffer element.
        """

        # Once the guard is released, write_idx will be placed into read_queue.
        return self.Guard(self.read_queue, self.arys, self.__put_idx)


Instruction:
Explain what the following Python 3 code does
def get_direct(self):

        read_idx = self.__get_idx()

        if read_idx is QueueClosed:
            return QueueClosed

        # Once the guard is released, read_idx will be placed into write_queue.
        return self.Guard(self.write_queue, self.arys, lambda: read_idx)

Response: Returns a direct access to the buffer element.


Instruction:
Implement a function in Python 3 to
close the queue signalling that no more data can be put into the queue.

Response:
def close(self):
        """Close the queue, signalling that no more data can be put into the queue."""
        self.read_queue.put(QueueClosed)
        self.write_queue.put(QueueClosed)


Instruction:
How would you explain what the following Python 3 function does
def __get_batch(self, path, length, last=False):
        import tables
        h5_file = tables.open_file(self.filename, 'r')
        h5_node = h5_file.get_node(path)

        if len(h5_node) == 0:
            raise Exception("Cannot read from empty dataset.")

        # If the length isn't specified, then fall back to default values.
        if length is None:
            chunkshape = h5_node.chunkshape
            # If the array isn't chunked, then try to make the block close to 128KB.
            if chunkshape is None:
                default_length = 128*2**10//h5_node[0].nbytes  # Divides by one row of the dataset.
                length = min(h5_node.shape[0], default_length)
            # If it is chunked, then use the chunkshape for best performance.
            else:
                length = chunkshape[0]

        if last:
            example = h5_node[length*(len(h5_node)//length):].copy()
        else:
            example = h5_node[:length].copy()

        h5_file.close()
        return example

Response: Get a batch of data from the HDF5 file at path.


Instruction:
Write a Python 3 script for
getting the remainder elements from the HDF5 file at the given path.

Response:
def get_remainder(self, path, block_size):
        """
        Get the remainder elements. These elements will not be read in the direct queue access cyclic=False mode.

        :param path: The HDF5 path to the dataset to be read.
        :param block_size: The block size is used to calculate which elements will remain.
        :return: A copy of the remainder elements as a numpy array.
        """
        return self.__get_batch(path, length=block_size, last=True)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_queue(self, path, n_procs=4, read_ahead=None, cyclic=False, block_size=None, ordered=False):
        # Get a block_size length of elements from the dataset to serve as a template for creating the buffer.
        # If block_size=None, then get_batch calculates an appropriate block size.
        example = self.__get_batch(path, block_size)
        block_size = example.shape[0]

        if read_ahead is None:
            # 2x No. of processes for writing, 1 extra for reading.
            read_ahead = 2*n_procs + 1

        cbuf = SharedCircBuf(read_ahead, example)
        stop = multiprocessing.Event()
        barrier = Barrier(n_procs)

        # If ordering has been requested, create a synchronizer.
        sync = GuardSynchronizer() if ordered else None

        procs = []
        for i in range(n_procs):
            # Each process is offset in the dataset by i*block_size
            # The skip length is set to n_procs*block_size so that no block is read by 2 processes.
            process = multiprocessing.Process(target=_Streamer__read_process, args=(
                self, path, block_size, cbuf, stop, barrier, cyclic,
                i * block_size, n_procs * block_size, sync
            ))
            process.daemon = True
            process.start()
            procs.append(process)

        # If the queue is not cyclic, then the cessation of reading data needs to be monitored.
        if not cyclic:

            # This closure defines a background thread that waits until all processes have finished.
            # At this point, all data from the dataset has been read, and the buffer is closed.
            def monitor():
                for p in procs:
                    p.join()
                cbuf.close()

            monitor_thread = threading.Thread(target=monitor)
            monitor_thread.daemon = True
            monitor_thread.start()

        return Streamer.Queue(cbuf, stop, block_size)

Response: Get a queue object that allows direct access to the internal buffer.


Instruction:
Given the following Python 3 function, write the documentation
def get_generator(self, path, *args, **kw_args):
        q = self.get_queue(path=path, *args, **kw_args)

        try:
            # This generator just implements a standard access pattern for the direct access queue.
            for guard in q.iter():
                with guard as batch:
                    batch_copy = batch.copy()

                for row in batch_copy:
                    yield row

            last_batch = self.get_remainder(path, q.block_size)
            for row in last_batch:
                yield row

        finally:
            q.close()

Response: Get a generator that yields the elements from the queue at a time at a time.


Instruction:
Here you have a function in Python 3, explain what it does
def parse(ifp, pb_cls, **kwargs):
    mode = 'rb'
    if isinstance(ifp, str):
        istream = open(ifp, mode=mode, **kwargs)
    else:
        istream = open(fileobj=ifp, mode=mode, **kwargs)
    with istream:
        for data in istream:
            pb_obj = pb_cls()
            pb_obj.ParseFromString(data)
            yield pb_obj

Response:
Parse a stream.
 article.