Instruction:
Create a Python 3 function for
trying to find a whitespace pattern in the existing parameters to be applied to a newly added parameter .

Response:
def find_whitespace_pattern(self):
        """
        Try to find a whitespace pattern in the existing parameters
        to be applied to a newly added parameter
        """
        name_ws = []
        value_ws = []
        for entry in self._entries:
            name_ws.append(get_whitespace(entry.name))
            if entry.value != '':
                value_ws.append(get_whitespace(entry._value))  # _value is unstripped
        if len(value_ws) >= 1:
            value_ws = most_common(value_ws)
        else:
            value_ws = ('', ' ')
        if len(name_ws) >= 1:
            name_ws = most_common(name_ws)
        else:
            name_ws = (' ', '')
        return name_ws, value_ws


Instruction:
Can you implement a function in Python 3 that
generates the path on disk for a specific project and date.

Response:
def _path_for_file(self, project_name, date):
        """
        Generate the path on disk for a specified project and date.

        :param project_name: the PyPI project name for the data
        :type project: str
        :param date: the date for the data
        :type date: datetime.datetime
        :return: path for where to store this data on disk
        :rtype: str
        """
        return os.path.join(
            self.cache_path,
            '%s_%s.json' % (project_name, date.strftime('%Y%m%d'))
        )


Instruction:
Can you generate the documentation for the following Python 3 function
def get(self, project, date):
        fpath = self._path_for_file(project, date)
        logger.debug('Cache GET project=%s date=%s - path=%s',
                     project, date.strftime('%Y-%m-%d'), fpath)
        try:
            with open(fpath, 'r') as fh:
                data = json.loads(fh.read())
        except:
            logger.debug('Error getting from cache for project=%s date=%s',
                         project, date.strftime('%Y-%m-%d'))
            return None
        data['cache_metadata']['date'] = datetime.strptime(
            data['cache_metadata']['date'],
            '%Y%m%d'
        )
        data['cache_metadata']['updated'] = datetime.fromtimestamp(
            data['cache_metadata']['updated']
        )
        return data

Response: Get the cache data for a specified project and date.


Instruction:
Given the following Python 3 function, write the documentation
def set(self, project, date, data, data_ts):
        data['cache_metadata'] = {
            'project': project,
            'date': date.strftime('%Y%m%d'),
            'updated': time.time(),
            'version': VERSION,
            'data_ts': data_ts
        }
        fpath = self._path_for_file(project, date)
        logger.debug('Cache SET project=%s date=%s - path=%s',
                     project, date.strftime('%Y-%m-%d'), fpath)
        with open(fpath, 'w') as fh:
            fh.write(json.dumps(data))

Response: Set the cache data for a specified project for the specified date.


Instruction:
Here you have a function in Python 3, explain what it does
def get_dates_for_project(self, project):
        file_re = re.compile(r'^%s_([0-9]{8})\.json$' % project)
        all_dates = []
        for f in os.listdir(self.cache_path):
            if not os.path.isfile(os.path.join(self.cache_path, f)):
                continue
            m = file_re.match(f)
            if m is None:
                continue
            all_dates.append(datetime.strptime(m.group(1), '%Y%m%d'))
        return sorted(all_dates)

Response: Return a list of the dates we have in cache for the specified project sorted in ascending date order.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_args(argv):
    p = argparse.ArgumentParser(
        description='pypi-download-stats - Calculate detailed download stats '
                    'and generate HTML and badges for PyPI packages - '
                    '<%s>' % PROJECT_URL,
        prog='pypi-download-stats'
    )
    p.add_argument('-V', '--version', action='version',
                   version='%(prog)s ' + VERSION)
    p.add_argument('-v', '--verbose', dest='verbose', action='count',
                   default=0,
                   help='verbose output. specify twice for debug-level output.')
    m = p.add_mutually_exclusive_group()
    m.add_argument('-Q', '--no-query', dest='query', action='store_false',
                   default=True, help='do not query; just generate output '
                                      'from cached data')
    m.add_argument('-G', '--no-generate', dest='generate', action='store_false',
                   default=True, help='do not generate output; just query '
                                      'data and cache results')
    p.add_argument('-o', '--out-dir', dest='out_dir', action='store', type=str,
                   default='./pypi-stats', help='output directory (default: '
                                                './pypi-stats')
    p.add_argument('-p', '--project-id', dest='project_id', action='store',
                   type=str, default=None,
                   help='ProjectID for your Google Cloud user, if not using '
                        'service account credentials JSON file')
    # @TODO this is tied to the DiskDataCache class
    p.add_argument('-c', '--cache-dir', dest='cache_dir', action='store',
                   type=str, default='./pypi-stats-cache',
                   help='stats cache directory (default: ./pypi-stats-cache)')
    p.add_argument('-B', '--backfill-num-days', dest='backfill_days', type=int,
                   action='store', default=7,
                   help='number of days of historical data to backfill, if '
                        'missing (defaut: 7). Note this may incur BigQuery '
                        'charges. Set to -1 to backfill all available history.')
    g = p.add_mutually_exclusive_group()
    g.add_argument('-P', '--project', dest='PROJECT', action='append', type=str,
                   help='project name to query/generate stats for (can be '
                        'specified more than once; '
                        'this will reduce query cost for multiple projects)')
    g.add_argument('-U', '--user', dest='user', action='store', type=str,
                   help='Run for all PyPI projects owned by the specified'
                        'user.')
    args = p.parse_args(argv)
    return args

Response: Parse command - line arguments.


Instruction:
Given the following Python 3 function, write the documentation
def set_log_level_format(level, format):
    formatter = logging.Formatter(fmt=format)
    logger.handlers[0].setFormatter(formatter)
    logger.setLevel(level)

Response: Set logger level and format.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _pypi_get_projects_for_user(username):
    client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
    pkgs = client.user_packages(username)  # returns [role, package]
    return [x[1] for x in pkgs]

Response:
Returns a list of all of the user s
    projects from the XMLRPC interface.


Instruction:
Make a summary of the following Python 3 code
def main(args=None):
    # parse args
    if args is None:
        args = parse_args(sys.argv[1:])

    # set logging level
    if args.verbose > 1:
        set_log_debug()
    elif args.verbose == 1:
        set_log_info()

    outpath = os.path.abspath(os.path.expanduser(args.out_dir))
    cachepath = os.path.abspath(os.path.expanduser(args.cache_dir))
    cache = DiskDataCache(cache_path=cachepath)

    if args.user:
        args.PROJECT = _pypi_get_projects_for_user(args.user)

    if args.query:
        DataQuery(args.project_id, args.PROJECT, cache).run_queries(
            backfill_num_days=args.backfill_days)
    else:
        logger.warning('Query disabled by command-line flag; operating on '
                       'cached data only.')
    if not args.generate:
        logger.warning('Output generation disabled by command-line flag; '
                       'exiting now.')
        raise SystemExit(0)
    for proj in args.PROJECT:
        logger.info('Generating output for: %s', proj)
        stats = ProjectStats(proj, cache)
        outdir = os.path.join(outpath, proj)
        OutputGenerator(proj, stats, outdir).generate()

Response:
Entry point for pypi
.


Instruction:
Can you tell what is the following Python 3 function doing
def generate_graph(self):
        logger.debug('Generating graph for %s', self._graph_id)
        # tools to use
        tools = [
            PanTool(),
            BoxZoomTool(),
            WheelZoomTool(),
            SaveTool(),
            ResetTool(),
            ResizeTool()
        ]

        # generate the stacked area graph
        try:
            g = Area(
                self._data, x='Date', y=self._y_series_names,
                title=self._title, stack=True, xlabel='Date',
                ylabel='Downloads', tools=tools,
                # note the width and height will be set by JavaScript
                plot_height=400, plot_width=800,
                toolbar_location='above', legend=False
            )
        except Exception as ex:
            logger.error("Error generating %s graph", self._graph_id)
            logger.error("Data: %s", self._data)
            logger.error("y=%s", self._y_series_names)
            raise ex

        lines = []
        legend_parts = []
        # add a line at the top of each Patch (stacked area) for hovertool
        for renderer in g.select(GlyphRenderer):
            if not isinstance(renderer.glyph, Patches):
                continue
            series_name = renderer.data_source.data['series'][0]
            logger.debug('Adding line for Patches %s (series: %s)', renderer,
                         series_name)
            line = self._line_for_patches(self._data, g, renderer, series_name)
            if line is not None:
                lines.append(line)
                legend_parts.append((series_name, [line]))

        # add the Hovertool, specifying only our line glyphs
        g.add_tools(
            HoverTool(
                tooltips=[
                    (self._y_name, '@SeriesName'),
                    ('Date', '@FmtDate'),
                    ('Downloads', '@Downloads'),
                ],
                renderers=lines,
                line_policy='nearest'
            )
        )

        # legend outside chart area
        legend = Legend(legends=legend_parts, location=(0, 0))
        g.add_layout(legend, 'right')
        return components(g)

Response: Generate the graph for the current object.


Instruction:
Create a Python 3 function for
adding a line along the top edge of a Patch in a stacked Area Chart; return the new Glyph for addition to HoverTool. :param data: original data for the graph :type data: dict :param chart: Chart to add the line to :type chart: bokeh.charts.Chart :param renderer: GlyphRenderer containing one Patches glyph, to draw the line for :type renderer: bokeh.models.renderers.GlyphRenderer :param series_name: the data series name this Patches represents :type series_name: str :return: GlyphRenderer for a Line at the top edge of this Patch :rtype: bokeh.models.renderers.GlyphRenderer

Response:
def _line_for_patches(self, data, chart, renderer, series_name):
        """
        Add a line along the top edge of a Patch in a stacked Area Chart; return
        the new Glyph for addition to HoverTool.

        :param data: original data for the graph
        :type data: dict
        :param chart: Chart to add the line to
        :type chart: bokeh.charts.Chart
        :param renderer: GlyphRenderer containing one Patches glyph, to draw
          the line for
        :type renderer: bokeh.models.renderers.GlyphRenderer
        :param series_name: the data series name this Patches represents
        :type series_name: str
        :return: GlyphRenderer for a Line at the top edge of this Patch
        :rtype: bokeh.models.renderers.GlyphRenderer
        """
        # @TODO this method needs a major refactor
        # get the original x and y values, and color
        xvals = deepcopy(renderer.data_source.data['x_values'][0])
        yvals = deepcopy(renderer.data_source.data['y_values'][0])
        line_color = renderer.glyph.fill_color

        # save original values for logging if needed
        orig_xvals = [x for x in xvals]
        orig_yvals = [y for y in yvals]

        # get a list of the values
        new_xvals = [x for x in xvals]
        new_yvals = [y for y in yvals]

        # so when a Patch is made, the first point is (0,0); trash it
        xvals = new_xvals[1:]
        yvals = new_yvals[1:]

        # then, we can tell the last point in the "top" line because it will be
        # followed by a point with the same x value and a y value of 0.
        last_idx = None
        for idx, val in enumerate(xvals):
            if yvals[idx+1] == 0 and xvals[idx+1] == xvals[idx]:
                last_idx = idx
                break

        if last_idx is None:
            logger.error('Unable to find top line of patch (x_values=%s '
                         'y_values=%s', orig_xvals, orig_yvals)
            return None

        # truncate our values to just what makes up the top line
        xvals = xvals[:last_idx+1]
        yvals = yvals[:last_idx+1]

        # Currently (bokeh 0.12.1) HoverTool won't show the tooltip for the last
        # point in our line. As a hack for this, add a point with the same Y
        # value and an X slightly before it.
        lastx = xvals[-1]
        xvals[-1] = lastx - 1000  # 1000 nanoseconds
        xvals.append(lastx)
        yvals.append(yvals[-1])
        # get the actual download counts from the original data
        download_counts = [
            data[series_name][y] for y in range(0, len(yvals) - 1)
        ]
        download_counts.append(download_counts[-1])

        # create a ColumnDataSource for the new overlay line
        data2 = {
            'x': xvals,  # Date/x values are numpy.datetime64
            'y': yvals,
            # the following are hacks for data that we want in the HoverTool
            # tooltip
            'SeriesName': [series_name for _ in yvals],
            # formatted date
            'FmtDate': [self.datetime64_to_formatted_date(x) for x in xvals],
            # to show the exact value, not where the pointer is
            'Downloads': download_counts
        }
        # set the formatted date for our hacked second-to-last point to the
        # same value as the last point
        data2['FmtDate'][-2] = data2['FmtDate'][-1]

        # create the CloumnDataSource, then the line for it, then the Glyph
        line_ds = ColumnDataSource(data2)
        line = Line(x='x', y='y', line_color=line_color)
        lineglyph = chart.add_glyph(line_ds, line)
        return lineglyph


Instruction:
How would you explain what the following Python 3 function does
def _get_cache_dates(self):
        all_dates = self.cache.get_dates_for_project(self.project_name)
        dates = []
        last_date = None
        for val in sorted(all_dates):
            if last_date is None:
                last_date = val
                continue
            if val - last_date > timedelta(hours=48):
                # reset dates to start from here
                logger.warning("Last cache date was %s, current date is %s; "
                               "delta is too large. Starting cache date series "
                               "at current date.", last_date, val)
                dates = []
            last_date = val
            dates.append(val)
        # find the first download record, and only look at dates after that
        for idx, cache_date in enumerate(dates):
            data = self._cache_get(cache_date)
            if not self._is_empty_cache_record(data):
                logger.debug("First cache date with data: %s", cache_date)
                return dates[idx:]
        return dates

Response: Get a list of dates that are present in the cache.


Instruction:
Create a Python 3 function to
return True if the specified cache record has no data False otherwise.

Response:
def _is_empty_cache_record(self, rec):
        """
        Return True if the specified cache record has no data, False otherwise.

        :param rec: cache record returned by :py:meth:`~._cache_get`
        :type rec: dict
        :return: True if record is empty, False otherwise
        :rtype: bool
        """
        # these are taken from DataQuery.query_one_table()
        for k in [
            'by_version',
            'by_file_type',
            'by_installer',
            'by_implementation',
            'by_system',
            'by_distro',
            'by_country'
        ]:
            if k in rec and len(rec[k]) > 0:
                return False
        return True


Instruction:
Here you have a function in Python 3, explain what it does
def _cache_get(self, date):
        if date in self.cache_data:
            logger.debug('Using class-cached data for date %s',
                         date.strftime('%Y-%m-%d'))
            return self.cache_data[date]
        logger.debug('Getting data from cache for date %s',
                     date.strftime('%Y-%m-%d'))
        data = self.cache.get(self.project_name, date)
        self.cache_data[date] = data
        return data

Response: Get the cache data for the specified day.


Instruction:
Can you tell what is the following Python 3 function doing
def _compound_column_value(k1, k2):
        k1 = ProjectStats._column_value(k1)
        k2 = ProjectStats._column_value(k2)
        if k1 == 'unknown' and k2 == 'unknown':
            return 'unknown'
        return '%s %s' % (k1, k2)

Response: Like _column_value but collapses two unknowns into one.


Instruction:
Create a Python 3 function for
shortening the version string to be a number of components.

Response:
def _shorten_version(ver, num_components=2):
        """
        If ``ver`` is a dot-separated string with at least (num_components +1)
        components, return only the first two. Else return the original string.

        :param ver: version string
        :type ver: str
        :return: shortened (major, minor) version
        :rtype: str
        """
        parts = ver.split('.')
        if len(parts) <= num_components:
            return ver
        return '.'.join(parts[:num_components])


Instruction:
Explain what the following Python 3 code does
def per_version_data(self):
        ret = {}
        for cache_date in self.cache_dates:
            data = self._cache_get(cache_date)
            if len(data['by_version']) == 0:
                data['by_version'] = {'other': 0}
            ret[cache_date] = data['by_version']
        return ret

Response: Return download data by version.


Instruction:
Create a Python 3 function for
returning download data by file type.

Response:
def per_file_type_data(self):
        """
        Return download data by file type.

        :return: dict of cache data; keys are datetime objects, values are
          dict of file type (str) to count (int)
        :rtype: dict
        """
        ret = {}
        for cache_date in self.cache_dates:
            data = self._cache_get(cache_date)
            if len(data['by_file_type']) == 0:
                data['by_file_type'] = {'other': 0}
            ret[cache_date] = data['by_file_type']
        return ret


Instruction:
How would you explain what the following Python 3 function does
def per_installer_data(self):
        ret = {}
        for cache_date in self.cache_dates:
            data = self._cache_get(cache_date)
            ret[cache_date] = {}
            for inst_name, inst_data in data['by_installer'].items():
                for inst_ver, count in inst_data.items():
                    k = self._compound_column_value(
                        inst_name,
                        self._shorten_version(inst_ver)
                    )
                    ret[cache_date][k] = count
            if len(ret[cache_date]) == 0:
                ret[cache_date]['unknown'] = 0
        return ret

Response: Return download data by installer name and version.


Instruction:
Write a Python 3 script for
returning download data by python impelementation name and version.

Response:
def per_implementation_data(self):
        """
        Return download data by python impelementation name and version.

        :return: dict of cache data; keys are datetime objects, values are
          dict of implementation name/version (str) to count (int).
        :rtype: dict
        """
        ret = {}
        for cache_date in self.cache_dates:
            data = self._cache_get(cache_date)
            ret[cache_date] = {}
            for impl_name, impl_data in data['by_implementation'].items():
                for impl_ver, count in impl_data.items():
                    k = self._compound_column_value(
                        impl_name,
                        self._shorten_version(impl_ver)
                    )
                    ret[cache_date][k] = count
            if len(ret[cache_date]) == 0:
                ret[cache_date]['unknown'] = 0
        return ret


Instruction:
Create a Python 3 function to
return download data by system.

Response:
def per_system_data(self):
        """
        Return download data by system.

        :return: dict of cache data; keys are datetime objects, values are
          dict of system (str) to count (int)
        :rtype: dict
        """
        ret = {}
        for cache_date in self.cache_dates:
            data = self._cache_get(cache_date)
            ret[cache_date] = {
                self._column_value(x): data['by_system'][x]
                for x in data['by_system']
            }
            if len(ret[cache_date]) == 0:
                ret[cache_date]['unknown'] = 0
        return ret


Instruction:
Create a Python 3 function to
return download data by country.

Response:
def per_country_data(self):
        """
        Return download data by country.

        :return: dict of cache data; keys are datetime objects, values are
          dict of country (str) to count (int)
        :rtype: dict
        """
        ret = {}
        for cache_date in self.cache_dates:
            data = self._cache_get(cache_date)
            ret[cache_date] = {}
            for cc, count in data['by_country'].items():
                k = '%s (%s)' % (self._alpha2_to_country(cc), cc)
                ret[cache_date][k] = count
            if len(ret[cache_date]) == 0:
                ret[cache_date]['unknown'] = 0
        return ret


Instruction:
Can you create a Python 3 function that
returns download data by distro name and version.

Response:
def per_distro_data(self):
        """
        Return download data by distro name and version.

        :return: dict of cache data; keys are datetime objects, values are
          dict of distro name/version (str) to count (int).
        :rtype: dict
        """
        ret = {}
        for cache_date in self.cache_dates:
            data = self._cache_get(cache_date)
            ret[cache_date] = {}
            for distro_name, distro_data in data['by_distro'].items():
                if distro_name.lower() == 'red hat enterprise linux server':
                    distro_name = 'RHEL'
                for distro_ver, count in distro_data.items():
                    ver = self._shorten_version(distro_ver, num_components=1)
                    if distro_name.lower() == 'os x':
                        ver = self._shorten_version(distro_ver,
                                                    num_components=2)
                    k = self._compound_column_value(distro_name, ver)
                    ret[cache_date][k] = count
            if len(ret[cache_date]) == 0:
                ret[cache_date]['unknown'] = 0
        return ret


Instruction:
Here you have a function in Python 3, explain what it does
def downloads_per_day(self):
        count, num_days = self._downloads_for_num_days(7)
        res = ceil(count / num_days)
        logger.debug("Downloads per day = (%d / %d) = %d", count, num_days, res)
        return res

Response:
Return the number of downloads per day average over the past 7 days
        of data.


Instruction:
Implement a function in Python 3 to
return the number of downloads per week in the last 7 days of data.

Response:
def downloads_per_week(self):
        """
        Return the number of downloads in the last 7 days.

        :return: number of downloads in the last 7 days; if we have less than
          7 days of data, returns None.
        :rtype: int
        """
        if len(self.cache_dates) < 7:
            logger.error("Only have %d days of data; cannot calculate "
                         "downloads per week", len(self.cache_dates))
            return None
        count, _ = self._downloads_for_num_days(7)
        logger.debug("Downloads per week = %d", count)
        return count


Instruction:
Write a Python 3 function that can
get the total number of downloads for a given number of days of data.

Response:
def _downloads_for_num_days(self, num_days):
        """
        Given a number of days of historical data to look at (starting with
        today and working backwards), return the total number of downloads
        for that time range, and the number of days of data we had (in cases
        where we had less data than requested).

        :param num_days: number of days of data to look at
        :type num_days: int
        :return: 2-tuple of (download total, number of days of data)
        :rtype: tuple
        """
        logger.debug("Getting download total for last %d days", num_days)
        dates = self.cache_dates
        logger.debug("Cache has %d days of data", len(dates))
        if len(dates) > num_days:
            dates = dates[(-1 * num_days):]
        logger.debug("Looking at last %d days of data", len(dates))
        dl_sum = 0
        for cache_date in dates:
            data = self._cache_get(cache_date)
            dl_sum += sum(data['by_version'].values())
        logger.debug("Sum of download counts: %d", dl_sum)
        return dl_sum, len(dates)


Instruction:
How would you explain what the following Python 3 function does
def _get_project_id(self):
        fpath = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', None)
        if fpath is None:
            raise Exception('ERROR: No project ID specified, and '
                            'GOOGLE_APPLICATION_CREDENTIALS env var is not set')
        fpath = os.path.abspath(os.path.expanduser(fpath))
        logger.debug('Reading credentials file at %s to get project_id', fpath)
        with open(fpath, 'r') as fh:
            cred_data = json.loads(fh.read())
        return cred_data['project_id']

Response: Get our projectId from the GOOGLE_APPLICATION_CREDENTIALS file.


Instruction:
How would you implement a function in Python 3 that
connects to the BigQuery service.

Response:
def _get_bigquery_service(self):
        """
        Connect to the BigQuery service.

        Calling ``GoogleCredentials.get_application_default`` requires that
        you either be running in the Google Cloud, or have the
        ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable set to the path
        to a credentials JSON file.

        :return: authenticated BigQuery service connection object
        :rtype: `googleapiclient.discovery.Resource <http://google.github.io/\
google-api-python-client/docs/epy/googleapiclient.discovery.\
Resource-class.html>`_
        """
        logger.debug('Getting Google Credentials')
        credentials = GoogleCredentials.get_application_default()
        logger.debug('Building BigQuery service instance')
        bigquery_service = build('bigquery', 'v2', credentials=credentials)
        return bigquery_service


Instruction:
Write a Python 3 script for
getting a list of PyPI downloads table IDs.

Response:
def _get_download_table_ids(self):
        """
        Get a list of PyPI downloads table (sharded per day) IDs.

        :return: list of table names (strings)
        :rtype: ``list``
        """
        all_table_names = []  # matching per-date table names
        logger.info('Querying for all tables in dataset')
        tables = self.service.tables()
        request = tables.list(projectId=self._PROJECT_ID,
                              datasetId=self._DATASET_ID)
        while request is not None:
            response = request.execute()
            # if the number of results is evenly divisible by the page size,
            # we may end up with a last response that has no 'tables' key,
            # and is empty.
            if 'tables' not in response:
                response['tables'] = []
            for table in response['tables']:
                if table['type'] != 'TABLE':
                    logger.debug('Skipping %s (type=%s)',
                                 table['tableReference']['tableId'],
                                 table['type'])
                    continue
                if not self._table_re.match(table['tableReference']['tableId']):
                    logger.debug('Skipping table with non-matching name: %s',
                                 table['tableReference']['tableId'])
                    continue
                all_table_names.append(table['tableReference']['tableId'])
            request = tables.list_next(previous_request=request,
                                       previous_response=response)
        return sorted(all_table_names)


Instruction:
Can you generate the documentation for the following Python 3 function
def _datetime_for_table_name(self, table_name):
        m = self._table_re.match(table_name)
        dt = datetime.strptime(m.group(1), '%Y%m%d')
        return dt

Response: Return a datetime object for the date of the data in the specified table name.


Instruction:
Can you implement a function in Python 3 that
runs one query against BigQuery and return the result.

Response:
def _run_query(self, query):
        """
        Run one query against BigQuery and return the result.

        :param query: the query to run
        :type query: str
        :return: list of per-row response dicts (key => value)
        :rtype: ``list``
        """
        query_request = self.service.jobs()
        logger.debug('Running query: %s', query)
        start = datetime.now()
        resp = query_request.query(
            projectId=self.project_id, body={'query': query}
        ).execute()
        duration = datetime.now() - start
        logger.debug('Query response (in %s): %s', duration, resp)
        if not resp['jobComplete']:
            logger.error('Error: query reported job not complete!')
        if int(resp['totalRows']) == 0:
            return []
        if int(resp['totalRows']) != len(resp['rows']):
            logger.error('Error: query reported %s total rows, but only '
                         'returned %d', resp['totalRows'], len(resp['rows']))
        data = []
        fields = [f['name'] for f in resp['schema']['fields']]
        for row in resp['rows']:
            d = {}
            for idx, val in enumerate(row['f']):
                d[fields[idx]] = val['v']
            data.append(d)
        return data


Instruction:
How would you code a function in Python 3 to
query the database to get the newest timestamp in the given table.

Response:
def _get_newest_ts_in_table(self, table_name):
        """
        Return the timestamp for the newest record in the given table.

        :param table_name: name of the table to query
        :type table_name: str
        :return: timestamp of newest row in table
        :rtype: int
        """
        logger.debug(
            'Querying for newest timestamp in table %s', table_name
        )
        q = "SELECT TIMESTAMP_TO_SEC(MAX(timestamp)) AS max_ts %s;" % (
            self._from_for_table(table_name)
        )
        res = self._run_query(q)
        ts = int(res[0]['max_ts'])
        logger.debug('Newest timestamp in table %s: %s', table_name, ts)
        return ts


Instruction:
Create a Python 3 function for
querying for download data broken down by installer for one day.

Response:
def _query_by_installer(self, table_name):
        """
        Query for download data broken down by installer, for one day.

        :param table_name: table name to query against
        :type table_name: str
        :return: dict of download information by installer; keys are project
          name, values are a dict of installer names to dicts of installer
          version to download count.
        :rtype: dict
        """
        logger.info('Querying for downloads by installer in table %s',
                    table_name)
        q = "SELECT file.project, details.installer.name, " \
            "details.installer.version, COUNT(*) as dl_count " \
            "%s " \
            "%s " \
            "GROUP BY file.project, details.installer.name, " \
            "details.installer.version;" % (
                self._from_for_table(table_name),
                self._where_for_projects
            )
        res = self._run_query(q)
        result = self._dict_for_projects()
        # iterate through results
        for row in res:
            # pointer to the per-project result dict
            proj = result[row['file_project']]
            # grab the name and version; change None to 'unknown'
            iname = row['details_installer_name']
            iver = row['details_installer_version']
            if iname not in proj:
                proj[iname] = {}
            if iver not in proj[iname]:
                proj[iname][iver] = 0
            proj[iname][iver] += int(row['dl_count'])
        return result


Instruction:
How would you code a function in Python 3 to
query for download data broken down by system for one day.

Response:
def _query_by_system(self, table_name):
        """
        Query for download data broken down by system, for one day.

        :param table_name: table name to query against
        :type table_name: str
        :return: dict of download information by system; keys are project name,
          values are a dict of system names to download count.
        :rtype: dict
        """
        logger.info('Querying for downloads by system in table %s',
                    table_name)
        q = "SELECT file.project, details.system.name, COUNT(*) as dl_count " \
            "%s " \
            "%s " \
            "GROUP BY file.project, details.system.name;" % (
                self._from_for_table(table_name),
                self._where_for_projects
            )
        res = self._run_query(q)
        result = self._dict_for_projects()
        for row in res:
            system = row['details_system_name']
            result[row['file_project']][system] = int(
                row['dl_count'])
        return result


Instruction:
How would you implement a function in Python 3 that
queries for download data broken down by OS distribution for one day.

Response:
def _query_by_distro(self, table_name):
        """
        Query for download data broken down by OS distribution, for one day.

        :param table_name: table name to query against
        :type table_name: str
        :return: dict of download information by distro; keys are project name,
          values are a dict of distro names to dicts of distro version to
          download count.
        :rtype: dict
        """
        logger.info('Querying for downloads by distro in table %s', table_name)
        q = "SELECT file.project, details.distro.name, " \
            "details.distro.version, COUNT(*) as dl_count " \
            "%s " \
            "%s " \
            "GROUP BY file.project, details.distro.name, " \
            "details.distro.version;" % (
                self._from_for_table(table_name),
                self._where_for_projects
            )
        res = self._run_query(q)
        result = self._dict_for_projects()
        # iterate through results
        for row in res:
            # pointer to the per-project result dict
            proj = result[row['file_project']]
            # grab the name and version; change None to 'unknown'
            dname = row['details_distro_name']
            dver = row['details_distro_version']
            if dname not in proj:
                proj[dname] = {}
            if dver not in proj[dname]:
                proj[dname][dver] = 0
            proj[dname][dver] += int(row['dl_count'])
        return result


Instruction:
Explain what the following Python 3 code does
def query_one_table(self, table_name):
        table_date = self._datetime_for_table_name(table_name)
        logger.info('Running all queries for date table: %s (%s)', table_name,
                    table_date.strftime('%Y-%m-%d'))
        final = self._dict_for_projects()
        try:
            data_timestamp = self._get_newest_ts_in_table(table_name)
        except HttpError as exc:
            try:
                content = json.loads(exc.content.decode('utf-8'))
                if content['error']['message'].startswith('Not found: Table'):
                    logger.error("Table %s not found; no data for that day",
                                 table_name)
                    return
            except:
                pass
            raise exc
        # data queries
        # note - ProjectStats._is_empty_cache_record() needs to know keys
        for name, func in {
            'by_version': self._query_by_version,
            'by_file_type': self._query_by_file_type,
            'by_installer': self._query_by_installer,
            'by_implementation': self._query_by_implementation,
            'by_system': self._query_by_system,
            'by_distro': self._query_by_distro,
            'by_country': self._query_by_country_code
        }.items():
            tmp = func(table_name)
            for proj_name in tmp:
                final[proj_name][name] = tmp[proj_name]
        # add to cache
        for proj_name in final:
            self.cache.set(proj_name, table_date, final[proj_name],
                           data_timestamp)

Response: Query all queries for a given table and update the cache.


Instruction:
Implement a Python 3 function for
returning True if we have cached data for the specified datetime. Return False otherwise.

Response:
def _have_cache_for_date(self, dt):
        """
        Return True if we have cached data for all projects for the specified
        datetime. Return False otherwise.

        :param dt: datetime to find cache for
        :type dt: datetime.datetime
        :return: True if we have cache for all projects for this date, False
          otherwise
        :rtype: bool
        """
        for p in self.projects:
            if self.cache.get(p, dt) is None:
                return False
        return True


Instruction:
How would you explain what the following Python 3 function does
def backfill_history(self, num_days, available_table_names):
        if num_days == -1:
            # skip the first date, under the assumption that data may be
            # incomplete
            logger.info('Backfilling all available history')
            start_table = available_table_names[1]
        else:
            logger.info('Backfilling %d days of history', num_days)
            start_table = available_table_names[-1 * num_days]
        start_date = self._datetime_for_table_name(start_table)
        end_table = available_table_names[-3]
        end_date = self._datetime_for_table_name(end_table)
        logger.debug(
            'Backfilling history from %s (%s) to %s (%s)', start_table,
            start_date.strftime('%Y-%m-%d'), end_table,
            end_date.strftime('%Y-%m-%d')
        )
        for days in range((end_date - start_date).days + 1):
            backfill_dt = start_date + timedelta(days=days)
            if self._have_cache_for_date(backfill_dt):
                logger.info('Cache present for all projects for %s; skipping',
                            backfill_dt.strftime('%Y-%m-%d'))
                continue
            backfill_table = self._table_name_for_datetime(backfill_dt)
            logger.info('Backfilling %s (%s)', backfill_table,
                        backfill_dt.strftime('%Y-%m-%d'))
            self.query_one_table(backfill_table)

Response: Backfill historical data for num_days days.


Instruction:
Can you create a Python 3 function that
runs the data queries for the specified projects.

Response:
def run_queries(self, backfill_num_days=7):
        """
        Run the data queries for the specified projects.

        :param backfill_num_days: number of days of historical data to backfill,
          if missing
        :type backfill_num_days: int
        """
        available_tables = self._get_download_table_ids()
        logger.debug('Found %d available download tables: %s',
                     len(available_tables), available_tables)
        today_table = available_tables[-1]
        yesterday_table = available_tables[-2]
        self.query_one_table(today_table)
        self.query_one_table(yesterday_table)
        self.backfill_history(backfill_num_days, available_tables)


Instruction:
Make a summary of the following Python 3 code
def filter_data_columns(data):
    keys = set()
    for dt, d in data.items():
        for k in d:
            keys.add(k)
    return sorted([x for x in keys])

Response: Given a dict of data such as those in ProjectStats attributes return a list of distinct column keys in sorted order.


Instruction:
Implement a Python 3 function for
generating the HTML for the specified graphs.

Response:
def _generate_html(self):
        """
        Generate the HTML for the specified graphs.

        :return:
        :rtype:
        """
        logger.debug('Generating templated HTML')
        env = Environment(
            loader=PackageLoader('pypi_download_stats', 'templates'),
            extensions=['jinja2.ext.loopcontrols'])
        env.filters['format_date_long'] = filter_format_date_long
        env.filters['format_date_ymd'] = filter_format_date_ymd
        env.filters['data_columns'] = filter_data_columns
        template = env.get_template('base.html')

        logger.debug('Rendering template')
        html = template.render(
            project=self.project_name,
            cache_date=self._stats.as_of_datetime,
            user=getuser(),
            host=platform_node(),
            version=VERSION,
            proj_url=PROJECT_URL,
            graphs=self._graphs,
            graph_keys=self.GRAPH_KEYS,
            resources=Resources(mode='inline').render(),
            badges=self._badges
        )
        logger.debug('Template rendered')
        return html


Instruction:
Explain what the following Python 3 code does
def _data_dict_to_bokeh_chart_data(self, data):
        labels = []
        # find all the data keys
        keys = set()
        for date in data:
            for k in data[date]:
                keys.add(k)
        # final output dict
        out_data = {}
        for k in keys:
            out_data[k] = []
        # transform the data; deal with sparse data
        for data_date, data_dict in sorted(data.items()):
            labels.append(data_date)
            for k in out_data:
                if k in data_dict:
                    out_data[k].append(data_dict[k])
                else:
                    out_data[k].append(0)
        return out_data, labels

Response:
Take a dictionary of data and return a 2 - tuple of data dict and x labels list
        usable by bokeh. charts.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _limit_data(self, data):
        if len(data.keys()) <= 10:
            logger.debug("Data has less than 10 keys; not limiting")
            return data
        # average last 7 days of each series
        avgs = {}
        for k in data:
            if len(data[k]) <= 7:
                vals = data[k]
            else:
                vals = data[k][-7:]
            avgs[k] = sum(vals) / len(vals)
        # hold state
        final_data = {}  # final data dict
        other = []  # values for dropped/'other' series
        count = 0  # iteration counter
        # iterate the sorted averages; either drop or keep
        for k in sorted(avgs, key=avgs.get, reverse=True):
            if count < 10:
                final_data[k] = data[k]
                logger.debug("Keeping data series %s (average over last 7 "
                             "days of data: %d", k, avgs[k])
            else:
                logger.debug("Adding data series %s to 'other' (average over "
                             "last 7 days of data: %d", k, avgs[k])
                other.append(data[k])
            count += 1
        # sum up the other data and add to final
        final_data['other'] = [sum(series) for series in zip(*other)]
        return final_data

Response: Limit the data to 10 series.


Instruction:
Implement a function in Python 3 to
generate a downloads graph for the given name title and stats_data.

Response:
def _generate_graph(self, name, title, stats_data, y_name):
        """
        Generate a downloads graph; append it to ``self._graphs``.

        :param name: HTML name of the graph, also used in ``self.GRAPH_KEYS``
        :type name: str
        :param title: human-readable title for the graph
        :type title: str
        :param stats_data: data dict from ``self._stats``
        :type stats_data: dict
        :param y_name: Y axis metric name
        :type y_name: str
        """
        logger.debug('Generating chart data for %s graph', name)
        orig_data, labels = self._data_dict_to_bokeh_chart_data(stats_data)
        data = self._limit_data(orig_data)
        logger.debug('Generating %s graph', name)
        script, div = FancyAreaGraph(
            name, '%s %s' % (self.project_name, title), data, labels,
            y_name).generate_graph()
        logger.debug('%s graph generated', name)
        self._graphs[name] = {
            'title': title,
            'script': script,
            'div': div,
            'raw_data': stats_data
        }


Instruction:
Write a Python 3 script for
generating download badges. Append them to self. _badges.

Response:
def _generate_badges(self):
        """
        Generate download badges. Append them to ``self._badges``.
        """
        daycount = self._stats.downloads_per_day
        day = self._generate_badge('Downloads', '%d/day' % daycount)
        self._badges['per-day'] = day
        weekcount = self._stats.downloads_per_week
        if weekcount is None:
            # we don't have enough data for week (or month)
            return
        week = self._generate_badge('Downloads', '%d/week' % weekcount)
        self._badges['per-week'] = week
        monthcount = self._stats.downloads_per_month
        if monthcount is None:
            # we don't have enough data for month
            return
        month = self._generate_badge('Downloads', '%d/month' % monthcount)
        self._badges['per-month'] = month


Instruction:
Create a Python 3 function for
generating one badge via shields. io.

Response:
def _generate_badge(self, subject, status):
        """
        Generate SVG for one badge via shields.io.

        :param subject: subject; left-hand side of badge
        :type subject: str
        :param status: status; right-hand side of badge
        :type status: str
        :return: badge SVG
        :rtype: str
        """
        url = 'https://img.shields.io/badge/%s-%s-brightgreen.svg' \
              '?style=flat&maxAge=3600' % (subject, status)
        logger.debug("Getting badge for %s => %s (%s)", subject, status, url)
        res = requests.get(url)
        if res.status_code != 200:
            raise Exception("Error: got status %s for shields.io badge: %s",
                            res.status_code, res.text)
        logger.debug('Got %d character response from shields.io', len(res.text))
        return res.text


Instruction:
Write a Python 3 function for
generating all output types and write to disk.

Response:
def generate(self):
        """
        Generate all output types and write to disk.
        """
        logger.info('Generating graphs')
        self._generate_graph(
            'by-version',
            'Downloads by Version',
            self._stats.per_version_data,
            'Version'
        )
        self._generate_graph(
            'by-file-type',
            'Downloads by File Type',
            self._stats.per_file_type_data,
            'File Type'
        )
        self._generate_graph(
            'by-installer',
            'Downloads by Installer',
            self._stats.per_installer_data,
            'Installer'
        )
        self._generate_graph(
            'by-implementation',
            'Downloads by Python Implementation/Version',
            self._stats.per_implementation_data,
            'Implementation/Version'
        )
        self._generate_graph(
            'by-system',
            'Downloads by System Type',
            self._stats.per_system_data,
            'System'
        )
        self._generate_graph(
            'by-country',
            'Downloads by Country',
            self._stats.per_country_data,
            'Country'
        )
        self._generate_graph(
            'by-distro',
            'Downloads by Distro',
            self._stats.per_distro_data,
            'Distro'
        )
        self._generate_badges()
        logger.info('Generating HTML')
        html = self._generate_html()
        html_path = os.path.join(self.output_dir, 'index.html')
        with open(html_path, 'wb') as fh:
            fh.write(html.encode('utf-8'))
        logger.info('HTML report written to %s', html_path)
        logger.info('Writing SVG badges')
        for name, svg in self._badges.items():
            path = os.path.join(self.output_dir, '%s.svg' % name)
            with open(path, 'w') as fh:
                fh.write(svg)
            logger.info('%s badge written to: %s', name, path)


Instruction:
Implement a Python 3 function for
returning a formatted version of the given datetime instance.

Response:
def datetime_format(desired_format, datetime_instance=None,  *args, **kwargs):
    """
    Replaces format style phrases (listed in the dt_exps dictionary)
    with this datetime instance's information.

    .. code :: python

        reusables.datetime_format("Hey, it's {month-full} already!")
        "Hey, it's March already!"

    :param desired_format: string to add datetime details too
    :param datetime_instance: datetime.datetime instance, defaults to 'now'
    :param args: additional args to pass to str.format
    :param kwargs: additional kwargs to pass to str format
    :return: formatted string
    """
    for strf, exp in datetime_regex.datetime.format.items():
        desired_format = exp.sub(strf, desired_format)
    if not datetime_instance:
        datetime_instance = now()
    return datetime_instance.strftime(desired_format.format(*args, **kwargs))


Instruction:
Can you generate the documentation for the following Python 3 function
def datetime_from_iso(iso_string):
    try:
        assert datetime_regex.datetime.datetime.match(iso_string).groups()[0]
    except (ValueError, AssertionError, IndexError, AttributeError):
        raise TypeError("String is not in ISO format")
    try:
        return datetime.datetime.strptime(iso_string, "%Y-%m-%dT%H:%M:%S.%f")
    except ValueError:
        return datetime.datetime.strptime(iso_string, "%Y-%m-%dT%H:%M:%S")

Response: Create a DateTime object from an ISO string


Instruction:
Write a Python 3 script for
getting a current DateTime object. By default is local.

Response:
def now(utc=False, tz=None):
    """
    Get a current DateTime object. By default is local.

    .. code:: python

        reusables.now()
        # DateTime(2016, 12, 8, 22, 5, 2, 517000)

        reusables.now().format("It's {24-hour}:{min}")
        # "It's 22:05"

    :param utc: bool, default False, UTC time not local
    :param tz: TimeZone as specified by the datetime module
    :return: reusables.DateTime
    """
    return datetime.datetime.utcnow() if utc else datetime.datetime.now(tz=tz)


Instruction:
Write a Python 3 script for
running a command and return a new process object.

Response:
def run(command, input=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
        timeout=None, copy_local_env=False, **kwargs):
    """
    Cross platform compatible subprocess with CompletedProcess return.

    No formatting or encoding is performed on the output of subprocess, so it's
    output will appear the same on each version / interpreter as before.

    .. code:: python

        reusables.run('echo "hello world!', shell=True)
        # CPython 3.6
        # CompletedProcess(args='echo "hello world!', returncode=0,
        #                  stdout=b'"hello world!\\r\\n', stderr=b'')
        #
        # PyPy 5.4 (Python 2.7.10)
        # CompletedProcess(args='echo "hello world!', returncode=0L,
        # stdout='"hello world!\\r\\n')

    Timeout is only usable in Python 3.X, as it was not implemented before then,
    a NotImplementedError will be raised if specified on 2.x version of Python.

    :param command: command to run, str if shell=True otherwise must be list
    :param input: send something `communicate`
    :param stdout: PIPE or None
    :param stderr: PIPE or None
    :param timeout: max time to wait for command to complete
    :param copy_local_env: Use all current ENV vars in the subprocess as well
    :param kwargs: additional arguments to pass to Popen
    :return: CompletedProcess class
    """
    if copy_local_env:
        # Copy local env first and overwrite with anything manually specified
        env = os.environ.copy()
        env.update(kwargs.get('env', {}))
    else:
        env = kwargs.get('env')

    if sys.version_info >= (3, 5):
        return subprocess.run(command, input=input, stdout=stdout,
                              stderr=stderr, timeout=timeout, env=env,
                              **kwargs)

    # Created here instead of root level as it should never need to be
    # manually created or referenced
    class CompletedProcess(object):
        """A backwards compatible near clone of subprocess.CompletedProcess"""

        def __init__(self, args, returncode, stdout=None, stderr=None):
            self.args = args
            self.returncode = returncode
            self.stdout = stdout
            self.stderr = stderr

        def __repr__(self):
            args = ['args={0!r}'.format(self.args),
                    'returncode={0!r}'.format(self.returncode),
                    'stdout={0!r}'.format(self.stdout) if self.stdout else '',
                    'stderr={0!r}'.format(self.stderr) if self.stderr else '']
            return "{0}({1})".format(type(self).__name__,
                                     ', '.join(filter(None, args)))

        def check_returncode(self):
            if self.returncode:
                if python_version < (2, 7):
                    raise subprocess.CalledProcessError(self.returncode,
                                                        self.args)
                raise subprocess.CalledProcessError(self.returncode,
                                                    self.args,
                                                    self.stdout)

    proc = subprocess.Popen(command, stdout=stdout, stderr=stderr,
                            env=env, **kwargs)
    if PY3:
        out, err = proc.communicate(input=input, timeout=timeout)
    else:
        if timeout:
            raise NotImplementedError("Timeout is only available on Python 3")
        out, err = proc.communicate(input=input)
    return CompletedProcess(command, proc.returncode, out, err)


Instruction:
Can you tell what is the following Python 3 function doing
def run_in_pool(target, iterable, threaded=True, processes=4,
                asynchronous=False, target_kwargs=None):
    """ Run a set of iterables to a function in a Threaded or MP Pool.

    .. code: python

        def func(a):
            return a + a

        reusables.run_in_pool(func, [1,2,3,4,5])
        # [1, 4, 9, 16, 25]

    :param target: function to run
    :param iterable: positional arg to pass to function
    :param threaded: Threaded if True multiprocessed if False
    :param processes: Number of workers
    :param asynchronous: will do map_async if True
    :param target_kwargs: Keyword arguments to set on the function as a partial
    :return: pool results
    """
    my_pool = pool.ThreadPool if threaded else pool.Pool

    if target_kwargs:
        target = partial(target, **target_kwargs if target_kwargs else None)

    p = my_pool(processes)
    try:
        results = (p.map_async(target, iterable) if asynchronous
                   else p.map(target, iterable))
    finally:
        p.close()
        p.join()
    return results

Response: Run a set of iterables to a function in a Threaded or MP Pool.


Instruction:
Given the following Python 3 function, write the documentation
def tree_view(dictionary, level=0, sep="|  "):
    return "".join(["{0}{1}\n{2}".format(sep * level, k,
                   tree_view(v, level + 1, sep=sep) if isinstance(v, dict)
                   else "") for k, v in dictionary.items()])

Response: View a dictionary as a tree.


Instruction:
Implement a function in Python 3 to
turn the Namespace and sub Namespaces back into a native python dictionary.

Response:
def to_dict(self, in_dict=None):
        """
        Turn the Namespace and sub Namespaces back into a native
        python dictionary.

        :param in_dict: Do not use, for self recursion
        :return: python dictionary of this Namespace
        """
        in_dict = in_dict if in_dict else self
        out_dict = dict()
        for k, v in in_dict.items():
            if isinstance(v, Namespace):
                v = v.to_dict()
            out_dict[k] = v
        return out_dict


Instruction:
Write a Python 3 script for
returning value of key as a list

Response:
def list(self, item, default=None, spliter=",", strip=True, mod=None):
        """ Return value of key as a list

        :param item: key of value to transform
        :param mod: function to map against list
        :param default: value to return if item does not exist
        :param spliter: character to split str on
        :param strip: clean the list with the `strip`
        :return: list of items
        """
        try:
            item = self.__getattr__(item)
        except AttributeError as err:
            if default is not None:
                return default
            raise err
        if strip:
            item = item.lstrip("[").rstrip("]")
        out = [x.strip() if strip else x for x in item.split(spliter)]
        if mod:
            return list(map(mod, out))
        return out


Instruction:
Create a Python 3 function for
downloading a given URL to either file or memory.

Response:
def download(url, save_to_file=True, save_dir=".", filename=None,
             block_size=64000, overwrite=False, quiet=False):
    """
    Download a given URL to either file or memory

    :param url: Full url (with protocol) of path to download
    :param save_to_file: boolean if it should be saved to file or not
    :param save_dir: location of saved file, default is current working dir
    :param filename: filename to save as
    :param block_size: download chunk size
    :param overwrite: overwrite file if it already exists
    :param quiet: boolean to turn off logging for function
    :return: save location (or content if not saved to file)
    """

    if save_to_file:
        if not filename:
            filename = safe_filename(url.split('/')[-1])
            if not filename:
                filename = "downloaded_at_{}.file".format(time.time())
        save_location = os.path.abspath(os.path.join(save_dir, filename))
        if os.path.exists(save_location) and not overwrite:
            logger.error("File {0} already exists".format(save_location))
            return False
    else:
        save_location = "memory"

    try:
        request = urlopen(url)
    except ValueError as err:
        if not quiet and "unknown url type" in str(err):
            logger.error("Please make sure URL is formatted correctly and"
                         " starts with http:// or other protocol")
        raise err
    except Exception as err:
        if not quiet:
            logger.error("Could not download {0} - {1}".format(url, err))
        raise err

    try:
        kb_size = int(request.headers["Content-Length"]) / 1024
    except Exception as err:
        if not quiet:
            logger.debug("Could not determine file size - {0}".format(err))
        file_size = "(unknown size)"
    else:
        file_size = "({0:.1f} {1})".format(*(kb_size, "KB") if kb_size < 9999
                                           else (kb_size / 1024, "MB"))

    if not quiet:
        logger.info("Downloading {0} {1} to {2}".format(url, file_size,
                                                         save_location))

    if save_to_file:
        with open(save_location, "wb") as f:
            while True:
                buffer = request.read(block_size)
                if not buffer:
                    break
                f.write(buffer)
        return save_location
    else:
        return request.read()


Instruction:
Create a Python 3 function for
returning a list of IP addresses for a given URL.

Response:
def url_to_ips(url, port=None, ipv6=False, connect_type=socket.SOCK_STREAM,
               proto=socket.IPPROTO_TCP, flags=0):
    """
    Provide a list of IP addresses, uses `socket.getaddrinfo`

    .. code:: python

        reusables.url_to_ips("example.com", ipv6=True)
        # ['2606:2800:220:1:248:1893:25c8:1946']

    :param url: hostname to resolve to IP addresses
    :param port: port to send to getaddrinfo
    :param ipv6: Return IPv6 address if True, otherwise IPv4
    :param connect_type: defaults to STREAM connection, can be 0 for all
    :param proto: defaults to TCP, can be 0 for all
    :param flags: additional flags to pass
    :return: list of resolved IPs
    """
    try:
        results = socket.getaddrinfo(url, port,
                                     (socket.AF_INET if not ipv6
                                      else socket.AF_INET6),
                                     connect_type,
                                     proto,
                                     flags)
    except socket.gaierror:
        logger.exception("Could not resolve hostname")
        return []

    return list(set([result[-1][0] for result in results]))


Instruction:
How would you implement a function in Python 3 that
resolves a hostname based off an IP address.

Response:
def ip_to_url(ip_addr):
    """
    Resolve a hostname based off an IP address.

    This is very limited and will
    probably not return any results if it is a shared IP address or an
    address with improperly setup DNS records.

    .. code:: python

        reusables.ip_to_url('93.184.216.34') # example.com
        # None

        reusables.ip_to_url('8.8.8.8')
        # 'google-public-dns-a.google.com'

    :param ip_addr: IP address to resolve to hostname
    :return: string of hostname or None
    """
    try:
        return socket.gethostbyaddr(ip_addr)[0]
    except (socket.gaierror, socket.herror):
        logger.exception("Could not resolve hostname")


Instruction:
Can you write a function in Python 3 where it
creates a background thread for httpd and serve forever

Response:
def start(self):
        """Create a background thread for httpd and serve 'forever'"""
        self._process = threading.Thread(target=self._background_runner)
        self._process.start()


Instruction:
Given the following Python 3 function, write the documentation
def get_stream_handler(stream=sys.stderr, level=logging.INFO,
                       log_format=log_formats.easy_read):
    """
    Returns a set up stream handler to add to a logger.

    :param stream: which stream to use, defaults to sys.stderr
    :param level: logging level to set handler at
    :param log_format: formatter to use
    :return: stream handler
    """
    sh = logging.StreamHandler(stream)
    sh.setLevel(level)
    sh.setFormatter(logging.Formatter(log_format))
    return sh

Response: Returns a set up stream handler to add to a logger.


Instruction:
Here you have a function in Python 3, explain what it does
def get_file_handler(file_path="out.log", level=logging.INFO,
                     log_format=log_formats.easy_read,
                     handler=logging.FileHandler,
                     **handler_kwargs):
    """
    Set up a file handler to add to a logger.

    :param file_path: file to write the log to, defaults to out.log
    :param level: logging level to set handler at
    :param log_format: formatter to use
    :param handler: logging handler to use, defaults to FileHandler
    :param handler_kwargs: options to pass to the handler
    :return: handler
    """
    fh = handler(file_path, **handler_kwargs)
    fh.setLevel(level)
    fh.setFormatter(logging.Formatter(log_format))
    return fh

Response: Create a file handler for the next available log file.


Instruction:
Implement a function in Python 3 to
set up a logger for the specified module.

Response:
def setup_logger(module_name=None, level=logging.INFO, stream=sys.stderr,
                 file_path=None, log_format=log_formats.easy_read,
                 suppress_warning=True):
    """
    Grabs the specified logger and adds wanted handlers to it. Will
    default to adding a stream handler.

    :param module_name: logger name to use
    :param level: logging level to set logger at
    :param stream: stream to log to, or None
    :param file_path: file path to log to, or None
    :param log_format: format to set the handlers to use
    :param suppress_warning: add a NullHandler if no other handler is specified
    :return: configured logger
    """
    new_logger = logging.getLogger(module_name)

    if stream:
        new_logger.addHandler(get_stream_handler(stream, level, log_format))
    elif not file_path and suppress_warning and not new_logger.handlers:
            new_logger.addHandler(logging.NullHandler())

    if file_path:
        new_logger.addHandler(get_file_handler(file_path, level, log_format))
    if level > 0:
        new_logger.setLevel(level)
    return new_logger


Instruction:
Can you implement a function in Python 3 that
adds a new stream handler to the specified logger

Response:
def add_stream_handler(logger=None, stream=sys.stderr, level=logging.INFO,
                       log_format=log_formats.easy_read):
    """
    Addes a newly created stream handler to the specified logger

    :param logger: logging name or object to modify, defaults to root logger
    :param stream: which stream to use, defaults to sys.stderr
    :param level: logging level to set handler at
    :param log_format: formatter to use
    """
    if not isinstance(logger, logging.Logger):
        logger = logging.getLogger(logger)

    logger.addHandler(get_stream_handler(stream, level, log_format))


Instruction:
How would you implement a function in Python 3 that
adds a new file handler to the specified logger

Response:
def add_file_handler(logger=None, file_path="out.log", level=logging.INFO,
                     log_format=log_formats.easy_read):
    """
    Addes a newly created file handler to the specified logger

    :param logger: logging name or object to modify, defaults to root logger
    :param file_path: path to file to log to
    :param level: logging level to set handler at
    :param log_format: formatter to use
    """
    if not isinstance(logger, logging.Logger):
        logger = logging.getLogger(logger)

    logger.addHandler(get_file_handler(file_path, level, log_format))


Instruction:
How would you explain what the following Python 3 function does
def add_rotating_file_handler(logger=None, file_path="out.log",
                              level=logging.INFO,
                              log_format=log_formats.easy_read,
                              max_bytes=10*sizes.mb, backup_count=5,
                              **handler_kwargs):
    """ Adds a rotating file handler to the specified logger.

    :param logger: logging name or object to modify, defaults to root logger
    :param file_path: path to file to log to
    :param level: logging level to set handler at
    :param log_format: log formatter
    :param max_bytes: Max file size in bytes before rotating
    :param backup_count: Number of backup files
    :param handler_kwargs: options to pass to the handler
    """
    if not isinstance(logger, logging.Logger):
        logger = logging.getLogger(logger)

    logger.addHandler(get_file_handler(file_path, level, log_format,
                                       handler=RotatingFileHandler,
                                       maxBytes=max_bytes,
                                       backupCount=backup_count,
                                       **handler_kwargs))

Response: Adds a rotating file handler to the specified logger.


Instruction:
Write a Python 3 function for
adding a timed rotating file handler to the specified logger.

Response:
def add_timed_rotating_file_handler(logger=None, file_path="out.log",
                                    level=logging.INFO,
                                    log_format=log_formats.easy_read,
                                    when='w0', interval=1, backup_count=5,
                                    **handler_kwargs):
    """ Adds a timed rotating file handler to the specified logger.
    Defaults to weekly rotation, with 5 backups.

    :param logger: logging name or object to modify, defaults to root logger
    :param file_path: path to file to log to
    :param level: logging level to set handler at
    :param log_format: log formatter
    :param when:
    :param interval:
    :param backup_count: Number of backup files
    :param handler_kwargs: options to pass to the handler
    """
    if not isinstance(logger, logging.Logger):
        logger = logging.getLogger(logger)

    logger.addHandler(get_file_handler(file_path, level, log_format,
                                       handler=TimedRotatingFileHandler,
                                       when=when,
                                       interval=interval,
                                       backupCount=backup_count,
                                       **handler_kwargs))


Instruction:
Can you implement a function in Python 3 that
removes only stream handlers from the specified logger

Response:
def remove_stream_handlers(logger=None):
    """
    Remove only stream handlers from the specified logger

    :param logger: logging name or object to modify, defaults to root logger
    """
    if not isinstance(logger, logging.Logger):
        logger = logging.getLogger(logger)

    new_handlers = []
    for handler in logger.handlers:
        # FileHandler is a subclass of StreamHandler so
        # 'if not a StreamHandler' does not work
        if (isinstance(handler, logging.FileHandler) or
            isinstance(handler, logging.NullHandler) or
            (isinstance(handler, logging.Handler) and not
                isinstance(handler, logging.StreamHandler))):
            new_handlers.append(handler)
    logger.handlers = new_handlers


Instruction:
Make a summary of the following Python 3 code
def remove_file_handlers(logger=None):
    if not isinstance(logger, logging.Logger):
        logger = logging.getLogger(logger)

    new_handlers = []
    for handler in logger.handlers:
        if isinstance(handler, logging.FileHandler):
            handler.close()
        else:
            new_handlers.append(handler)
    logger.handlers = new_handlers

Response: Removes only file handlers from the specified logger.


Instruction:
Here you have a function in Python 3, explain what it does
def remove_all_handlers(logger=None):
    if not isinstance(logger, logging.Logger):
        logger = logging.getLogger(logger)

    remove_file_handlers(logger)
    logger.handlers = []

Response: Safely remove all handlers from the logger


Instruction:
Can you create a Python 3 function that
changes the levels of the logger and all handlers to the specified level.

Response:
def change_logger_levels(logger=None, level=logging.DEBUG):
    """
    Go through the logger and handlers and update their levels to the
    one specified.

    :param logger: logging name or object to modify, defaults to root logger
    :param level: logging level to set at (10=Debug, 20=Info, 30=Warn, 40=Error)
    """
    if not isinstance(logger, logging.Logger):
        logger = logging.getLogger(logger)

    logger.setLevel(level)
    for handler in logger.handlers:
        handler.level = level


Instruction:
How would you code a function in Python 3 to
find the names of all loggers currently registered in the hierarchy.

Response:
def get_registered_loggers(hide_children=False, hide_reusables=False):
    """
    Find the names of all loggers currently registered

    :param hide_children: only return top level logger names
    :param hide_reusables: hide the reusables loggers
    :return: list of logger names
    """

    return [logger for logger in logging.Logger.manager.loggerDict.keys()
            if not (hide_reusables and "reusables" in logger)
            and not (hide_children and "." in logger)]


Instruction:
Create a Python 3 function for
wrapping to make sure a function is only run once at a time.

Response:
def lock_it(lock=g_lock):
    """
    Wrapper. Simple wrapper to make sure a function is only run once at a time.

    .. code: python

        import reusables
        import time

        def func_one(_):
            time.sleep(5)

        @reusables.lock_it()
        def func_two(_):
            time.sleep(5)

        @reusables.time_it(message="test_1 took {0:.2f} seconds")
        def test_1():
            reusables.run_in_pool(func_one, (1, 2, 3), threaded=True)

        @reusables.time_it(message="test_2 took {0:.2f} seconds")
        def test_2():
            reusables.run_in_pool(func_two, (1, 2, 3), threaded=True)

        test_1()
        test_2()

        # test_1 took 5.04 seconds
        # test_2 took 15.07 seconds

    :param lock: Which lock to use, uses unique default
    """
    def func_wrapper(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            with lock:
                return func(*args, **kwargs)
        return wrapper
    return func_wrapper


Instruction:
Given the following Python 3 function, write the documentation
def time_it(log=None, message=None, append=None):
    def func_wrapper(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            # Can't use nonlocal in 2.x
            msg = (message if message else
                   "Function '{func}' took a total of {seconds} seconds")
            if not message:
                msg = _add_args(msg, *args, **kwargs)

            time_func = (time.perf_counter if python_version >= (3, 3)
                         else time.time)
            start_time = time_func()
            try:
                return func(*args, **kwargs)
            finally:
                total_time = time_func() - start_time

                time_string = msg.format(func=func.__name__,
                                         seconds=total_time,
                                         args=args, kwargs=kwargs)
                if log:
                    my_logger = logging.getLogger(log) if isinstance(log, str)\
                                else logger
                    my_logger.info(time_string)
                else:
                    print(time_string)
                if isinstance(append, list):
                    append.append(total_time)
        return wrapper
    return func_wrapper

Response:
Time the amount of time it takes the execution of the function
 ArcGIS and print it.


Instruction:
Write a Python 3 function for
wrapping for reusables. queue_it.

Response:
def queue_it(queue=g_queue, **put_args):
    """
    Wrapper. Instead of returning the result of the function, add it to a queue.

    .. code: python

        import reusables
        import queue

        my_queue = queue.Queue()

        @reusables.queue_it(my_queue)
        def func(a):
            return a

        func(10)

        print(my_queue.get())
        # 10

    :param queue: Queue to add result into
    """
    def func_wrapper(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            queue.put(func(*args, **kwargs), **put_args)
        return wrapper
    return func_wrapper


Instruction:
How would you implement a function in Python 3 that
catches exceptions and returns the specified default or sent to a handler function instead.

Response:
def catch_it(exceptions=(Exception, ), default=None, handler=None):
    """
    If the function encounters an exception, catch it, and
    return the specified default or sent to a handler function instead.

    .. code :: python

        def handle_error(exception, func, *args, **kwargs):
            print(f"{func.__name__} raised {exception} when called with {args}")

        @reusables.catch_it(handler=err_func)
        def will_raise(message="Hello")
            raise Exception(message)

    :param exceptions: tuple of exceptions to catch
    :param default: what to return if the exception is caught
    :param handler: function to send exception, func, *args and **kwargs
    """
    def func_wrapper(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except exceptions as err:
                if handler:
                    return handler(err, func, *args, **kwargs)
                return default
        return wrapper
    return func_wrapper


Instruction:
Write a Python 3 function for
retrying a function if an exception is raised or if output_check returns False.

Response:
def retry_it(exceptions=(Exception, ), tries=10, wait=0, handler=None,
             raised_exception=ReusablesError, raised_message=None):
    """
    Retry a function if an exception is raised, or if output_check returns
    False.

    Message format options: {func} {args} {kwargs}

    :param exceptions: tuple of exceptions to catch
    :param tries: number of tries to retry the function
    :param wait: time to wait between executions in seconds
    :param handler: function to check if output is valid, must return bool
    :param raised_exception: default is ReusablesError
    :param raised_message: message to pass to raised exception
    """
    def func_wrapper(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            msg = (raised_message if raised_message
                   else "Max retries exceeded for function '{func}'")
            if not raised_message:
                msg = _add_args(msg, *args, **kwargs)
            try:
                result = func(*args, **kwargs)
            except exceptions:
                if tries:
                    if wait:
                        time.sleep(wait)
                    return retry_it(exceptions=exceptions, tries=tries-1,
                                    handler=handler,
                                    wait=wait)(func)(*args, **kwargs)
                if raised_exception:
                    exc = raised_exception(msg.format(func=func.__name__,
                                           args=args, kwargs=kwargs))
                    exc.__cause__ = None
                    raise exc
            else:
                if handler:
                    if not handler(result):
                        return retry_it(exceptions=exceptions, tries=tries - 1,
                                        handler=handler,
                                        wait=wait)(func)(*args, **kwargs)
                return result
        return wrapper
    return func_wrapper


Instruction:
How would you explain what the following Python 3 function does
def extract(archive_file, path=".", delete_on_success=False,
            enable_rar=False):
    """
    Automatically detect archive type and extract all files to specified path.

    .. code:: python

        import os

        os.listdir(".")
        # ['test_structure.zip']

        reusables.extract("test_structure.zip")

        os.listdir(".")
        # [ 'test_structure', 'test_structure.zip']

    :param archive_file: path to file to extract
    :param path: location to extract to
    :param delete_on_success: Will delete the original archive if set to True
    :param enable_rar: include the rarfile import and extract
    :return: path to extracted files
    """

    if not os.path.exists(archive_file) or not os.path.getsize(archive_file):
        logger.error("File {0} unextractable".format(archive_file))
        raise OSError("File does not exist or has zero size")

    arch = None
    if zipfile.is_zipfile(archive_file):
        logger.debug("File {0} detected as a zip file".format(archive_file))
        arch = zipfile.ZipFile(archive_file)
    elif tarfile.is_tarfile(archive_file):
        logger.debug("File {0} detected as a tar file".format(archive_file))
        arch = tarfile.open(archive_file)
    elif enable_rar:
        import rarfile
        if rarfile.is_rarfile(archive_file):
            logger.debug("File {0} detected as "
                         "a rar file".format(archive_file))
            arch = rarfile.RarFile(archive_file)

    if not arch:
        raise TypeError("File is not a known archive")

    logger.debug("Extracting files to {0}".format(path))

    try:
        arch.extractall(path=path)
    finally:
        arch.close()

    if delete_on_success:
        logger.debug("Archive {0} will now be deleted".format(archive_file))
        os.unlink(archive_file)

    return os.path.abspath(path)

Response: Extract the contents of the archive file to specified path.


Instruction:
Implement a Python 3 function for
archiving a list of files or folders in a single archive.

Response:
def archive(files_to_archive, name="archive.zip", archive_type=None,
            overwrite=False, store=False, depth=None, err_non_exist=True,
            allow_zip_64=True, **tarfile_kwargs):
    """ Archive a list of files (or files inside a folder), can chose between

        - zip
        - tar
        - gz (tar.gz, tgz)
        - bz2 (tar.bz2)

    .. code:: python

        reusables.archive(['reusables', '.travis.yml'],
                              name="my_archive.bz2")
        # 'C:\\Users\\Me\\Reusables\\my_archive.bz2'

    :param files_to_archive: list of files and folders to archive
    :param name: path and name of archive file
    :param archive_type: auto-detects unless specified
    :param overwrite: overwrite if archive exists
    :param store: zipfile only, True will not compress files
    :param depth: specify max depth for folders
    :param err_non_exist: raise error if provided file does not exist
    :param allow_zip_64: must be enabled for zip files larger than 2GB
    :param tarfile_kwargs: extra args to pass to tarfile.open
    :return: path to created archive
    """
    if not isinstance(files_to_archive, (list, tuple)):
        files_to_archive = [files_to_archive]

    if not archive_type:
        if name.lower().endswith("zip"):
            archive_type = "zip"
        elif name.lower().endswith("gz"):
            archive_type = "gz"
        elif name.lower().endswith("z2"):
            archive_type = "bz2"
        elif name.lower().endswith("tar"):
            archive_type = "tar"
        else:
            err_msg = ("Could not determine archive "
                       "type based off {0}".format(name))
            logger.error(err_msg)
            raise ValueError(err_msg)
        logger.debug("{0} file detected for {1}".format(archive_type, name))
    elif archive_type not in ("tar", "gz", "bz2", "zip"):
        err_msg = ("archive_type must be zip, gz, bz2,"
                   " or gz, was {0}".format(archive_type))
        logger.error(err_msg)
        raise ValueError(err_msg)

    if not overwrite and os.path.exists(name):
        err_msg = "File {0} exists and overwrite not specified".format(name)
        logger.error(err_msg)
        raise OSError(err_msg)

    if archive_type == "zip":
        arch = zipfile.ZipFile(name, 'w',
                               zipfile.ZIP_STORED if store else
                               zipfile.ZIP_DEFLATED,
                               allowZip64=allow_zip_64)
        write = arch.write
    elif archive_type in ("tar", "gz", "bz2"):
        mode = archive_type if archive_type != "tar" else ""
        arch = tarfile.open(name, 'w:{0}'.format(mode), **tarfile_kwargs)
        write = arch.add
    else:
        raise ValueError("archive_type must be zip, gz, bz2, or gz")

    try:
        for file_path in files_to_archive:
            if os.path.isfile(file_path):
                if err_non_exist and not os.path.exists(file_path):
                    raise OSError("File {0} does not exist".format(file_path))
                write(file_path)
            elif os.path.isdir(file_path):
                for nf in find_files(file_path, abspath=False, depth=depth):
                    write(nf)
    except (Exception, KeyboardInterrupt) as err:
        logger.exception("Could not archive {0}".format(files_to_archive))
        try:
            arch.close()
        finally:
            os.unlink(name)
        raise err
    else:
        arch.close()

    return os.path.abspath(name)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def list_to_csv(my_list, csv_file):
    if PY3:
        csv_handler = open(csv_file, 'w', newline='')
    else:
        csv_handler = open(csv_file, 'wb')

    try:
        writer = csv.writer(csv_handler, delimiter=',', quoting=csv.QUOTE_ALL)
        writer.writerows(my_list)
    finally:
        csv_handler.close()

Response: Save a list of lists to a CSV file.


Instruction:
Create a Python 3 function for
opening and transform a CSV file into a list of lists.

Response:
def csv_to_list(csv_file):
    """
    Open and transform a CSV file into a matrix (list of lists).

    .. code:: python

        reusables.csv_to_list("example.csv")
        # [['Name', 'Location'],
        #  ['Chris', 'South Pole'],
        #  ['Harry', 'Depth of Winter'],
        #  ['Bob', 'Skull']]

    :param csv_file: Path to CSV file as str
    :return: list
    """
    with open(csv_file, 'r' if PY3 else 'rb') as f:
        return list(csv.reader(f))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def load_json(json_file, **kwargs):
    with open(json_file) as f:
        return json.load(f, **kwargs)

Response: Open and load data from a JSON file.


Instruction:
Write a Python 3 function that can
save a dictionary to a JSON file.

Response:
def save_json(data, json_file, indent=4, **kwargs):
    """
    Takes a dictionary and saves it to a file as JSON

    .. code:: python

        my_dict = {"key_1": "val_1",
                   "key_for_dict": {"sub_dict_key": 8}}

        reusables.save_json(my_dict,"example.json")

    example.json

    .. code::

        {
            "key_1": "val_1",
            "key_for_dict": {
                "sub_dict_key": 8
            }
        }

    :param data: dictionary to save as JSON
    :param json_file: Path to save file location as str
    :param indent: Format the JSON file with so many numbers of spaces
    :param kwargs: Additional arguments for the json.dump command
    """
    with open(json_file, "w") as f:
        json.dump(data, f, indent=indent, **kwargs)


Instruction:
Explain what the following Python 3 code does
def config_dict(config_file=None, auto_find=False, verify=True, **cfg_options):
    if not config_file:
        config_file = []

    cfg_parser = ConfigParser.ConfigParser(**cfg_options)
    cfg_files = []

    if config_file:
        if not isinstance(config_file, (list, tuple)):
            if isinstance(config_file, str):
                cfg_files.append(config_file)
            else:
                raise TypeError("config_files must be a list or a string")
        else:
            cfg_files.extend(config_file)

    if auto_find:
        cfg_files.extend(find_files_list(
            current_root if isinstance(auto_find, bool) else auto_find,
            ext=(".cfg", ".config", ".ini")))

    logger.info("config files to be used: {0}".format(cfg_files))

    if verify:
        cfg_parser.read([cfg for cfg in cfg_files if os.path.exists(cfg)])
    else:
        cfg_parser.read(cfg_files)

    return dict((section, dict(cfg_parser.items(section)))
                for section in cfg_parser.sections())

Response: Return configuration options as dictionary. Accepts either a single config file or a list of files.


Instruction:
Can you create a Python 3 function that
returns the configuration options as a Namespace.

Response:
def config_namespace(config_file=None, auto_find=False,
                     verify=True, **cfg_options):
    """
    Return configuration options as a Namespace.

    .. code:: python

        reusables.config_namespace(os.path.join("test", "data",
                                                "test_config.ini"))
        # <Namespace: {'General': {'example': 'A regul...>

    :param config_file: path or paths to the files location
    :param auto_find: look for a config type file at this location or below
    :param verify: make sure the file exists before trying to read
    :param cfg_options: options to pass to the parser
    :return: Namespace of the config files
    """
    return ConfigNamespace(**config_dict(config_file, auto_find,
                                         verify, **cfg_options))


Instruction:
Can you tell what is the following Python 3 function doing
def _walk(directory, enable_scandir=False, **kwargs):
    walk = os.walk
    if python_version < (3, 5) and enable_scandir:
        import scandir
        walk = scandir.walk
    return walk(directory, **kwargs)

Response:
Internal function to return a walk generator from os. walk
    on Python < 3. 5


Instruction:
Can you generate a brief explanation for the following Python 3 code
def os_tree(directory, enable_scandir=False):
    if not os.path.exists(directory):
        raise OSError("Directory does not exist")
    if not os.path.isdir(directory):
        raise OSError("Path is not a directory")

    full_list = []
    for root, dirs, files in _walk(directory, enable_scandir=enable_scandir):
        full_list.extend([os.path.join(root, d).lstrip(directory) + os.sep
                          for d in dirs])
    tree = {os.path.basename(directory): {}}
    for item in full_list:
        separated = item.split(os.sep)
        is_dir = separated[-1:] == ['']
        if is_dir:
            separated = separated[:-1]
        parent = tree[os.path.basename(directory)]
        for index, path in enumerate(separated):
            if path in parent:
                parent = parent[path]
                continue
            else:
                parent[path] = dict()
            parent = parent[path]
    return tree

Response: Returns a dictionary of the contents of a directory as a dictionary hierarchy.


Instruction:
Implement a function in Python 3 to
hash a given file with md5 or any other and return the hex digest.

Response:
def file_hash(path, hash_type="md5", block_size=65536, hex_digest=True):
    """
    Hash a given file with md5, or any other and return the hex digest. You
    can run `hashlib.algorithms_available` to see which are available on your
    system unless you have an archaic python version, you poor soul).

    This function is designed to be non memory intensive.

    .. code:: python

        reusables.file_hash(test_structure.zip")
        # '61e387de305201a2c915a4f4277d6663'

    :param path: location of the file to hash
    :param hash_type: string name of the hash to use
    :param block_size: amount of bytes to add to hasher at a time
    :param hex_digest: returned as hexdigest, false will return digest
    :return: file's hash
    """
    hashed = hashlib.new(hash_type)
    with open(path, "rb") as infile:
        buf = infile.read(block_size)
        while len(buf) > 0:
            hashed.update(buf)
            buf = infile.read(block_size)
    return hashed.hexdigest() if hex_digest else hashed.digest()


Instruction:
How would you implement a function in Python 3 that
finds all files in a directory and return an iterator of all files that match the requirements.

Response:
def find_files(directory=".", ext=None, name=None,
               match_case=False, disable_glob=False, depth=None,
               abspath=False, enable_scandir=False):
    """
    Walk through a file directory and return an iterator of files
    that match requirements. Will autodetect if name has glob as magic
    characters.

    Note: For the example below, you can use find_files_list to return as a
    list, this is simply an easy way to show the output.

    .. code:: python

        list(reusables.find_files(name="ex", match_case=True))
        # ['C:\\example.pdf',
        #  'C:\\My_exam_score.txt']

        list(reusables.find_files(name="*free*"))
        # ['C:\\my_stuff\\Freedom_fight.pdf']

        list(reusables.find_files(ext=".pdf"))
        # ['C:\\Example.pdf',
        #  'C:\\how_to_program.pdf',
        #  'C:\\Hunks_and_Chicks.pdf']

        list(reusables.find_files(name="*chris*"))
        # ['C:\\Christmas_card.docx',
        #  'C:\\chris_stuff.zip']

    :param directory: Top location to recursively search for matching files
    :param ext: Extensions of the file you are looking for
    :param name: Part of the file name
    :param match_case: If name or ext has to be a direct match or not
    :param disable_glob: Do not look for globable names or use glob magic check
    :param depth: How many directories down to search
    :param abspath: Return files with their absolute paths
    :param enable_scandir: on python < 3.5 enable external scandir package
    :return: generator of all files in the specified directory
    """
    if ext or not name:
        disable_glob = True
    if not disable_glob:
        disable_glob = not glob.has_magic(name)

    if ext and isinstance(ext, str):
        ext = [ext]
    elif ext and not isinstance(ext, (list, tuple)):
        raise TypeError("extension must be either one extension or a list")
    if abspath:
        directory = os.path.abspath(directory)
    starting_depth = directory.count(os.sep)

    for root, dirs, files in _walk(directory, enable_scandir=enable_scandir):
        if depth and root.count(os.sep) - starting_depth >= depth:
            continue

        if not disable_glob:
            if match_case:
                raise ValueError("Cannot use glob and match case, please "
                                 "either disable glob or not set match_case")
            glob_generator = glob.iglob(os.path.join(root, name))
            for item in glob_generator:
                yield item
            continue

        for file_name in files:
            if ext:
                for end in ext:
                    if file_name.lower().endswith(end.lower() if not
                    match_case else end):
                        break
                else:
                    continue
            if name:
                if match_case and name not in file_name:
                    continue
                elif name.lower() not in file_name.lower():
                    continue
            yield os.path.join(root, file_name)


Instruction:
Can you write a function in Python 3 where it
removes all empty directories from a path.

Response:
def remove_empty_directories(root_directory, dry_run=False, ignore_errors=True,
                             enable_scandir=False):
    """
    Remove all empty folders from a path. Returns list of empty directories.

    :param root_directory: base directory to start at
    :param dry_run: just return a list of what would be removed
    :param ignore_errors: Permissions are a pain, just ignore if you blocked
    :param enable_scandir: on python < 3.5 enable external scandir package
    :return: list of removed directories
    """
    listdir = os.listdir
    if python_version < (3, 5) and enable_scandir:
        import scandir as _scandir

        def listdir(directory):
            return list(_scandir.scandir(directory))

    directory_list = []
    for root, directories, files in _walk(root_directory,
                                          enable_scandir=enable_scandir,
                                          topdown=False):
        if (not directories and not files and os.path.exists(root) and
                    root != root_directory and os.path.isdir(root)):
            directory_list.append(root)
            if not dry_run:
                try:
                    os.rmdir(root)
                except OSError as err:
                    if ignore_errors:
                        logger.info("{0} could not be deleted".format(root))
                    else:
                        raise err
        elif directories and not files:
            for directory in directories:
                directory = join_paths(root, directory, strict=True)
                if (os.path.exists(directory) and os.path.isdir(directory) and
                        not listdir(directory)):
                    directory_list.append(directory)
                    if not dry_run:
                        try:
                            os.rmdir(directory)
                        except OSError as err:
                            if ignore_errors:
                                logger.info("{0} could not be deleted".format(
                                    directory))
                            else:
                                raise err
    return directory_list