Instruction:
How would you explain what the following Python 3 function does
def _update_fw_pointers(collection, symbol, version, previous_version, is_append, shas_to_add=None):
    version[FW_POINTERS_CONFIG_KEY] = ARCTIC_FORWARD_POINTERS_CFG.name  # get the str as enum is not BSON serializable

    if ARCTIC_FORWARD_POINTERS_CFG is FwPointersCfg.DISABLED:
        return

    version_shas = set()

    if is_append:
        # Appends are tricky, as we extract the SHAs from the previous version (assuming it has FW pointers info)
        prev_fw_cfg = get_fwptr_config(previous_version)
        if prev_fw_cfg is FwPointersCfg.DISABLED.name:
            version_shas.update(Binary(sha) for sha in collection.find(
                {'symbol': symbol,
                 'parent': version_base_or_id(previous_version),
                 'segment': {'$lt': previous_version['up_to']}},
                {'sha': 1}))
        else:
            version_shas.update(previous_version[FW_POINTERS_REFS_KEY])

    # It is a write (we always get the all-inclusive set of SHAs), so no need to obtain previous SHAs
    version_shas.update(shas_to_add)

    # Verify here the number of seen segments vs expected ones
    if len(version_shas) != version['segment_count']:
        raise pymongo.errors.OperationFailure("Mismatched number of forward pointers to segments for {}: {} != {})"
                                              "Is append: {}. Previous version: {}. "
                                              "Gathered forward pointers segment shas: {}.".format(
            symbol, len(version_shas), version['segment_count'], is_append, previous_version['_id'], version_shas))

    version[FW_POINTERS_REFS_KEY] = list(version_shas)

Response: This function updates the version document with the forward pointers to segments.


Instruction:
Given the following Python 3 function, write the documentation
def _spec_fw_pointers_aware(symbol, version, from_index=None, to_index=None):
    spec = {'symbol': symbol,
            'segment': {'$lt': version['up_to'] if to_index is None else to_index}}
    if from_index is not None:
        spec['segment']['$gte'] = from_index

    # Version was written with an older arctic, not FW-pointers aware, or written with FW pointers disabled
    if FW_POINTERS_CONFIG_KEY not in version or version[FW_POINTERS_CONFIG_KEY] == FwPointersCfg.DISABLED.name:
        spec['parent'] = version_base_or_id(version)
        return spec

    v_fw_config = FwPointersCfg[version[FW_POINTERS_CONFIG_KEY]]

    # Version was created exclusively with fw pointers
    if v_fw_config is FwPointersCfg.ENABLED:
        if from_index is None and to_index is None:
            del spec['segment']
        spec['sha'] = {'$in': version[FW_POINTERS_REFS_KEY]}
        return spec

    # Version was created both with fw and legacy pointers, choose based on module configuration
    if v_fw_config is FwPointersCfg.HYBRID:
        if ARCTIC_FORWARD_POINTERS_CFG is FwPointersCfg.DISABLED:
            spec['parent'] = version_base_or_id(version)
        else:
            if from_index is None and to_index is None:
                del spec['segment']
            spec['sha'] = {'$in': version[FW_POINTERS_REFS_KEY]}
        return spec

    # The code below shouldn't really be reached.
    raise DataIntegrityException("Unhandled FW pointers configuration ({}: {}/{}/{})".format(
        version.get('symbol'), version.get('_id'), version.get('version'), v_fw_config))

Response: This method updates the find query filter spec used to read a version.


Instruction:
Write a Python 3 function for
returning the index and the index of the array from which the symbol is from.

Response:
def _index_range(self, version, symbol, from_version=None, **kwargs):
        """
        Tuple describing range to read from the ndarray - closed:open
        """
        from_index = None
        if from_version:
            from_index = from_version['up_to']
        return from_index, None


Instruction:
Given the following Python 3 function, write the documentation
def _do_read(self, collection, version, symbol, index_range=None):
        from_index = index_range[0] if index_range else None
        to_index = version['up_to']
        if index_range and index_range[1] and index_range[1] < version['up_to']:
            to_index = index_range[1]
        segment_count = version.get('segment_count') if from_index is None else None

        spec = _spec_fw_pointers_aware(symbol, version, from_index, to_index)

        data = bytearray()
        i = -1
        for i, x in enumerate(sorted(collection.find(spec), key=itemgetter('segment'))):
            data.extend(decompress(x['data']) if x['compressed'] else x['data'])

        # Check that the correct number of segments has been returned
        if segment_count is not None and i + 1 != segment_count:
            raise OperationFailure("Incorrect number of segments returned for {}:{}.  Expected: {}, but got {}. {}".format(
                                   symbol, version['version'], segment_count, i + 1, collection.database.name + '.' + collection.name))

        dtype = self._dtype(version['dtype'], version.get('dtype_metadata', {}))
        rtn = np.frombuffer(data, dtype=dtype).reshape(version.get('shape', (-1)))
        return rtn

Response: Internal method to read the data from the database.


Instruction:
Write a Python 3 script to
add the library with the given date range to the underlying collection of libraries used by this store.

Response:
def add(self, date_range, library_name):
        """
        Adds the library with the given date range to the underlying collection of libraries used by this store.
        The underlying libraries should not overlap as the date ranges are assumed to be CLOSED_CLOSED by this function
        and the rest of the class.

        Arguments:

        date_range: A date range provided on the assumption that it is CLOSED_CLOSED. If for example the underlying
        libraries were split by year, the start of the date range would be datetime.datetime(year, 1, 1) and the end
        would be datetime.datetime(year, 12, 31, 23, 59, 59, 999000). The date range must fall on UTC day boundaries,
        that is the start must be add midnight and the end must be 1 millisecond before midnight.

        library_name: The name of the underlying library. This must be the name of a valid Arctic library
        """
        # check that the library is valid
        try:
            self._arctic_lib.arctic[library_name]
        except Exception as e:
            logger.error("Could not load library")
            raise e
        assert date_range.start and date_range.end, "Date range should have start and end properties {}".format(date_range)
        start = date_range.start.astimezone(mktz('UTC')) if date_range.start.tzinfo is not None else date_range.start.replace(tzinfo=mktz('UTC'))
        end = date_range.end.astimezone(mktz('UTC')) if date_range.end.tzinfo is not None else date_range.end.replace(tzinfo=mktz('UTC'))
        assert start.time() == time.min and end.time() == end_time_min, "Date range should fall on UTC day boundaries {}".format(date_range)
        # check that the date range does not overlap
        library_metadata = self._get_library_metadata(date_range)
        if len(library_metadata) > 1 or (len(library_metadata) == 1 and library_metadata[0] != library_name):
            raise OverlappingDataException("""There are libraries that overlap with the date range:
library: {}
overlapping libraries: {}""".format(library_name, [l.library for l in library_metadata]))
        self._collection.update_one({'library_name': library_name},
                                    {'$set': {'start': start, 'end': end}}, upsert=True)


Instruction:
How would you explain what the following Python 3 function does
def write(self, symbol, data):

        # get the full set of date ranges that we have
        cursor = self._collection.find()
        for res in cursor:
            library = self._arctic_lib.arctic[res['library_name']]
            dslice = self._slice(data, to_dt(res['start'], mktz('UTC')), to_dt(res['end'], mktz('UTC')))
            if len(dslice) != 0:
                library.write(symbol, dslice)

Response: Writes the data to the underlying low - level library.


Instruction:
Write a Python 3 script to
retrieve the library metadata for the given date range.

Response:
def _get_library_metadata(self, date_range):
        """
        Retrieve the libraries for the given date range, the assumption is that the date ranges do not overlap and
        they are CLOSED_CLOSED.

        At the moment the date range is mandatory
        """
        if date_range is None:
            raise Exception("A date range must be provided")
        if not (date_range.start and date_range.end):
            raise Exception("The date range {0} must contain a start and end date".format(date_range))

        start = date_range.start if date_range.start.tzinfo is not None else date_range.start.replace(tzinfo=mktz())
        end = date_range.end if date_range.end.tzinfo is not None else date_range.end.replace(tzinfo=mktz())
        query = {'$or': [{'start': {'$lte': start}, 'end': {'$gte': start}},
                         {'start': {'$gte': start}, 'end': {'$lte': end}},
                         {'start': {'$lte': end}, 'end': {'$gte': end}}]}

        cursor = self._collection.find(query,
                                       projection={'library_name': 1, 'start': 1, 'end': 1},
                                       sort=[('start', pymongo.ASCENDING)])

        results = []
        for res in cursor:
            start = res['start']
            if date_range.start.tzinfo is not None and start.tzinfo is None:
                start = start.replace(tzinfo=mktz("UTC")).astimezone(tz=date_range.start.tzinfo)

            end = res['end']
            if date_range.end.tzinfo is not None and end.tzinfo is None:
                end = end.replace(tzinfo=mktz("UTC")).astimezone(tz=date_range.end.tzinfo)

            results.append(TickStoreLibrary(res['library_name'], DateRange(start, end, CLOSED_CLOSED)))
        return results


Instruction:
Create a Python 3 function for
writing data to the version store.

Response:
def write(self, symbol, data, prune_previous_version=True, metadata=None, **kwargs):
        """
        Records a write request to be actioned on context exit. Takes exactly the same parameters as the regular
        library write call.
        """
        if data is not None:
            # We only write data if existing data is None or the Timeseries data has changed or metadata has changed
            if self.base_ts.data is None or not are_equals(data, self.base_ts.data) or metadata != self.base_ts.metadata:
                self._do_write = True
        self._write = partial(self._version_store.write, symbol, data, prune_previous_version=prune_previous_version,
                              metadata=metadata, **kwargs)


Instruction:
Write a Python 3 function that can
convert a pandas DataFrame to records.

Response:
def _to_records(self, df, string_max_len=None, forced_dtype=None):
        """
        Similar to DataFrame.to_records()
        Differences:
            Attempt type conversion for pandas columns stored as objects (e.g. strings),
            as we can only store primitives in the ndarray.
            Use dtype metadata to store column and index names.

        string_max_len: integer - enforces a string size on the dtype, if any
                                  strings exist in the record
        """

        index_names, ix_vals, metadata = self._index_to_records(df)
        columns, column_vals, multi_column = self._column_data(df)

        if "" in columns:
            raise ArcticException("Cannot use empty string as a column name.")

        if multi_column is not None:
            metadata['multi_column'] = multi_column
        metadata['columns'] = columns
        names = index_names + columns

        arrays = []
        for arr, name in zip(ix_vals + column_vals, index_names + columns):
            arrays.append(_to_primitive(arr, string_max_len,
                                        forced_dtype=None if forced_dtype is None else forced_dtype[name]))

        if forced_dtype is None:
            dtype = np.dtype([(str(x), v.dtype) if len(v.shape) == 1 else (str(x), v.dtype, v.shape[1])
                              for x, v in zip(names, arrays)],
                             metadata=metadata)
        else:
            dtype = forced_dtype

        # The argument names is ignored when dtype is passed
        rtn = np.rec.fromarrays(arrays, dtype=dtype, names=names)
        # For some reason the dtype metadata is lost in the line above
        # and setting rtn.dtype to dtype does not preserve the metadata
        # see https://github.com/numpy/numpy/issues/6771

        return (rtn, dtype)


Instruction:
How would you explain what the following Python 3 function does
def fast_check_serializable(self, df):
        i_dtype, f_dtypes = df.index.dtype, df.dtypes
        index_has_object = df.index.dtype is NP_OBJECT_DTYPE
        fields_with_object = [f for f in df.columns if f_dtypes[f] is NP_OBJECT_DTYPE]
        if df.empty or (not index_has_object and not fields_with_object):
            arr, _ = self._to_records(df.iloc[:10])  # only first few rows for performance
            return arr, {}
        # If only the Index has Objects, choose a small slice (two columns if possible,
        # to avoid switching from a DataFrame to a Series)
        df_objects_only = df[fields_with_object if fields_with_object else df.columns[:2]]
        # Let any exceptions bubble up from here
        arr, dtype = self._to_records(df_objects_only)
        return arr, {f: dtype[f] for f in dtype.names}

Response: Convert efficiently the frame s object - columns object - index multi - column columns or multi - column columns to records.


Instruction:
Can you write a function in Python 3 where it
applies func to each chunk in lib. symbol

Response:
def read_apply(lib, symbol, func, chunk_range=None):
    """
    Apply `func` to each chunk in lib.symbol

    Parameters
    ----------
    lib: arctic library
    symbol: str
        the symbol for the given item in the DB
    chunk_range: None, or a range object
        allows you to subset the chunks by range

    Returns
    -------
    generator
    """
    for chunk in lib.iterator(symbol, chunk_range=chunk_range):
        yield func(chunk)


Instruction:
Given the following Python 3 function, write the documentation
def list_symbols(self, regex=None, as_of=None, **kwargs):

        # Skip aggregation pipeline
        if not (regex or as_of or kwargs):
            return self.distinct('symbol')

        # Index-based query part
        index_query = {}
        if as_of is not None:
            index_query['start_time'] = {'$lte': as_of}

        if regex or as_of:
            # make sure that symbol is present in query even if only as_of is specified to avoid document scans
            # see 'Pipeline Operators and Indexes' at
            # https://docs.mongodb.com/manual/core/aggregation-pipeline/#aggregation-pipeline-operators-and-performance
            index_query['symbol'] = {'$regex': regex or '^'}

        # Document query part
        data_query = {}
        if kwargs:
            for k, v in six.iteritems(kwargs):
                data_query['metadata.' + k] = v

        # Sort using index, relying on https://docs.mongodb.com/manual/core/aggregation-pipeline-optimization/
        pipeline = [{'$sort': {'symbol': pymongo.ASCENDING,
                               'start_time': pymongo.DESCENDING}}]

        # Index-based filter on symbol and start_time
        if index_query:
            pipeline.append({'$match': index_query})
        # Group by 'symbol' and get the latest known data
        pipeline.append({'$group': {'_id': '$symbol',
                                    'metadata': {'$first': '$metadata'}}})
        # Match the data fields
        if data_query:
            pipeline.append({'$match': data_query})
        # Return only 'symbol' field value
        pipeline.append({'$project': {'_id': 0, 'symbol': '$_id'}})

        return sorted(r['symbol'] for r in self.aggregate(pipeline))

Response: Returns the list of symbols in this library.


Instruction:
Can you tell what is the following Python 3 function doing
def read_history(self, symbol):
        find = self.find({'symbol': symbol}, sort=[('start_time', pymongo.ASCENDING)])
        times = []
        entries = []
        for item in find:
            times.append(item['start_time'])
            entries.append(item['metadata'])
        return pd.DataFrame({symbol: entries}, times)

Response: Return all metadata saved for symbol


Instruction:
Given the following Python 3 function, write the documentation
def read(self, symbol, as_of=None):
        if as_of is not None:
            res = self.find_one({'symbol': symbol, 'start_time': {'$lte': as_of}},
                                sort=[('start_time', pymongo.DESCENDING)])
        else:
            res = self.find_one({'symbol': symbol}, sort=[('start_time', pymongo.DESCENDING)])
        return res['metadata'] if res is not None else None

Response: Reads the current metadata for a given item.


Instruction:
Make a summary of the following Python 3 code
def write_history(self, collection):
        documents = []
        for dataframe in collection:
            if len(dataframe.columns) != 1:
                raise ValueError('More than one symbol found in a DataFrame')
            symbol = dataframe.columns[0]
            times = dataframe.index
            entries = dataframe[symbol].values
            if self.has_symbol(symbol):
                self.purge(symbol)
            doc = {'symbol': symbol, 'metadata': entries[0], 'start_time': times[0]}
            for metadata, start_time in zip(entries[1:], times[1:]):
                if metadata == doc['metadata']:
                    continue
                doc['end_time'] = start_time
                documents.append(doc)
                doc = {'symbol': symbol, 'metadata': metadata, 'start_time': start_time}
            documents.append(doc)

        self.insert_many(documents)

Response: Write all metadata history for symbols in collection.


Instruction:
Implement a function in Python 3 to
append a new entry to the metadata table for the item.

Response:
def append(self, symbol, metadata, start_time=None):
        """
        Update metadata entry for `symbol`

        Parameters
        ----------
        symbol : `str`
            symbol name for the item
        metadata : `dict`
            to be persisted
        start_time : `datetime.datetime`
            when metadata becomes effective
            Default: datetime.datetime.utcnow()
        """
        if start_time is None:
            start_time = dt.utcnow()
        old_metadata = self.find_one({'symbol': symbol}, sort=[('start_time', pymongo.DESCENDING)])
        if old_metadata is not None:
            if old_metadata['start_time'] >= start_time:
                raise ValueError('start_time={} is earlier than the last metadata @{}'.format(start_time,
                                                                                old_metadata['start_time']))
            if old_metadata['metadata'] == metadata:
                return old_metadata
        elif metadata is None:
            return

        self.find_one_and_update({'symbol': symbol}, {'$set': {'end_time': start_time}},
                                 sort=[('start_time', pymongo.DESCENDING)])
        document = {'_id': bson.ObjectId(), 'symbol': symbol, 'metadata': metadata, 'start_time': start_time}
        mongo_retry(self.insert_one)(document)

        logger.debug('Finished writing metadata for %s', symbol)
        return document


Instruction:
Write a Python 3 script to
delete the last metadata of a specific symbol

Response:
def pop(self, symbol):
        """
        Delete current metadata of `symbol`

        Parameters
        ----------
        symbol : `str`
            symbol name to delete

        Returns
        -------
        Deleted metadata
        """
        last_metadata = self.find_one({'symbol': symbol}, sort=[('start_time', pymongo.DESCENDING)])
        if last_metadata is None:
            raise NoDataFoundException('No metadata found for symbol {}'.format(symbol))

        self.find_one_and_delete({'symbol': symbol}, sort=[('start_time', pymongo.DESCENDING)])
        mongo_retry(self.find_one_and_update)({'symbol': symbol}, {'$unset': {'end_time': ''}},
                                              sort=[('start_time', pymongo.DESCENDING)])

        return last_metadata


Instruction:
Write a Python 3 function that can
delete all metadata of symbol from the store.

Response:
def purge(self, symbol):
        """
        Delete all metadata of `symbol`

        Parameters
        ----------
        symbol : `str`
            symbol name to delete
        """
        logger.warning("Deleting entire metadata history for %r from %r" % (symbol, self._arctic_lib.get_name()))
        self.delete_many({'symbol': symbol})


Instruction:
Can you tell what is the following Python 3 function doing
def stats(self):
        res = {}
        db = self._collection.database
        res['dbstats'] = db.command('dbstats')
        res['data'] = db.command('collstats', self._collection.name)
        res['totals'] = {'count': res['data']['count'],
                         'size': res['data']['size']
                         }
        return res

Response: Return database usage statistics. Used by quota.


Instruction:
Can you implement a function in Python 3 that
stores the given thing in the database.

Response:
def store(self, thing):
        """
        Simple persistence method
        """
        to_store = {'field1': thing.field1,
                    'date_field': thing.date_field,
                    }
        to_store['stuff'] = Binary(cPickle.dumps(thing.stuff))
        # Respect any soft-quota on write - raises if stats().totals.size > quota
        self._arctic_lib.check_quota()
        self._collection.insert_one(to_store)


Instruction:
Write a Python 3 script for
reading the data for the named symbol and returns a BitemporalItem object with the data and metadata elements.

Response:
def read(self, symbol, as_of=None, raw=False, **kwargs):
        # TODO: shall we block from_version from getting into super.read?
        """Read data for the named symbol. Returns a BitemporalItem object with
        a data and metdata element (as passed into write).

        Parameters
        ----------
        symbol : `str`
            symbol name for the item
        as_of : `datetime.datetime`
            Return the data as it was as_of the point in time.
        raw : `bool`
            If True, will return the full bitemporal dataframe (i.e. all versions of the data). This also means as_of is
            ignored.

        Returns
        -------
        BitemporalItem namedtuple which contains a .data and .metadata element
        """
        item = self._store.read(symbol, **kwargs)
        last_updated = max(item.data.index.get_level_values(self.observe_column))
        if raw:
            return BitemporalItem(symbol=symbol, library=self._store._arctic_lib.get_name(), data=item.data,
                                  metadata=item.metadata,
                                  last_updated=last_updated)
        else:
            index_names = list(item.data.index.names)
            index_names.remove(self.observe_column)
            return BitemporalItem(symbol=symbol, library=self._store._arctic_lib.get_name(),
                                  data=groupby_asof(item.data, as_of=as_of, dt_col=index_names,
                                                    asof_col=self.observe_column),
                                  metadata=item.metadata, last_updated=last_updated)


Instruction:
Can you implement a function in Python 3 that
updates the data for the specified item in this library.

Response:
def update(self, symbol, data, metadata=None, upsert=True, as_of=None, **kwargs):
        """ Append 'data' under the specified 'symbol' name to this library.

        Parameters
        ----------
        symbol : `str`
            symbol name for the item
        data : `pd.DataFrame`
            to be persisted
        metadata : `dict`
            An optional dictionary of metadata to persist along with the symbol. If None and there are existing
            metadata, current metadata will be maintained
        upsert : `bool`
            Write 'data' if no previous version exists.
        as_of : `datetime.datetime`
            The "insert time". Default to datetime.now()
        """
        local_tz = mktz()
        if not as_of:
            as_of = dt.now()
        if as_of.tzinfo is None:
            as_of = as_of.replace(tzinfo=local_tz)
        data = self._add_observe_dt_index(data, as_of)
        if upsert and not self._store.has_symbol(symbol):
            df = data
        else:
            existing_item = self._store.read(symbol, **kwargs)
            if metadata is None:
                metadata = existing_item.metadata
            df = existing_item.data.append(data).sort_index(kind='mergesort')
        self._store.write(symbol, df, metadata=metadata, prune_previous_version=True)


Instruction:
Given the following Python 3 function, write the documentation
def string_to_daterange(str_range, delimiter='-', as_dates=False, interval=CLOSED_CLOSED):
    num_dates = str_range.count(delimiter) + 1
    if num_dates > 2:
        raise ValueError('Too many dates in input string [%s] with delimiter (%s)' % (str_range, delimiter))

    # Allow the user to use the [date-date), etc. range syntax to specify the interval.
    range_mode = Ranges.get(str_range[0] + str_range[-1], None)
    if range_mode:
        return string_to_daterange(str_range[1:-1], delimiter, as_dates, interval=range_mode)

    if as_dates:
        parse_dt = lambda s: parse(s).date() if s else None
    else:
        parse_dt = lambda s: parse(s) if s else None
    if num_dates == 2:
        d = [parse_dt(x) for x in str_range.split(delimiter)]
        oc = interval
    else:
        start = parse_dt(str_range)
        d = [start, start + datetime.timedelta(1)]
        oc = CLOSED_OPEN  # Always use closed-open for a single date/datetime.
    return DateRange(d[0], d[1], oc)

Response: Convert a string to a DateRange type.


Instruction:
How would you explain what the following Python 3 function does
def to_dt(date, default_tz=None):
    if isinstance(date, (int, long)):
        return ms_to_datetime(date, default_tz)
    elif date.tzinfo is None:
        if default_tz is None:
            raise ValueError("Must specify a TimeZone on incoming data")
        return date.replace(tzinfo=default_tz)
    return date

Response: Converts a datetime object to a non - naive datetime object.


Instruction:
Write a Python 3 script to
convert a date_range object to a Pandas DateRange slicing is CLOSED - CLOSED inclusive at both ends.

Response:
def to_pandas_closed_closed(date_range, add_tz=True):
    """
    Pandas DateRange slicing is CLOSED-CLOSED inclusive at both ends.

    Parameters
    ----------
    date_range : `DateRange` object
        converted to CLOSED_CLOSED form for Pandas slicing

    add_tz : `bool`
        Adds a TimeZone to the daterange start and end if it doesn't
        have one.

    Returns
    -------
    Returns a date_range with start-end suitable for slicing in pandas.
    """
    if not date_range:
        return None

    start = date_range.start
    end = date_range.end
    if start:
        start = to_dt(start, mktz()) if add_tz else start
        if date_range.startopen:
            start += timedelta(milliseconds=1)

    if end:
        end = to_dt(end, mktz()) if add_tz else end
        if date_range.endopen:
            end -= timedelta(milliseconds=1)
    return DateRange(start, end)


Instruction:
Given the following Python 3 function, write the documentation
def ms_to_datetime(ms, tzinfo=None):
    if not isinstance(ms, (int, long)):
        raise TypeError('expected integer, not %s' % type(ms))

    if tzinfo is None:
        tzinfo = mktz()

    return datetime.datetime.fromtimestamp(ms * 1e-3, tzinfo)

Response: Convert a millisecond time value to an offset - aware Python datetime object.


Instruction:
How would you code a function in Python 3 to
convert a Python datetime object to a millisecond epoch time value.

Response:
def datetime_to_ms(d):
    """Convert a Python datetime object to a millisecond epoch (UTC) time value."""
    try:
        millisecond = d.microsecond // 1000
        return calendar.timegm(_add_tzone(d).utctimetuple()) * 1000 + millisecond
    except AttributeError:
        raise TypeError('expect Python datetime object, not %s' % type(d))


Instruction:
Can you tell what is the following Python 3 function doing
def utc_dt_to_local_dt(dtm):
    utc_zone = mktz("UTC")
    if dtm.tzinfo is not None and dtm.tzinfo != utc_zone:
        raise ValueError(
            "Expected dtm without tzinfo or with UTC, not %r" % (
                dtm.tzinfo
            )
        )

    if dtm.tzinfo is None:
        dtm = dtm.replace(tzinfo=utc_zone)
    return dtm.astimezone(mktz())

Response: Convert a UTC datetime to a local datetime in local timezone


Instruction:
Can you generate a brief explanation for the following Python 3 code
def intersection(self, other):
        startopen = other.startopen if self.start is None \
            else self.startopen if other.start is None \
            else other.startopen if self.start < other.start \
            else self.startopen if self.start > other.start \
            else (self.startopen or other.startopen)
        endopen = other.endopen if self.end is None \
            else self.endopen if other.end is None \
            else other.endopen if self.end > other.end \
            else self.endopen if self.end < other.end \
            else (self.endopen or other.endopen)

        new_start = self.start if other.start is None \
            else other.start if self.start is None \
            else max(self.start, other.start)
        new_end = self.end if other.end is None \
            else other.end if self.end is None \
            else min(self.end, other.end)

        interval = INTERVAL_LOOKUP[(startopen, endopen)]

        return DateRange(new_start, new_end, interval)

Response: Create a new DateRange representing the maximal range enclosed by this range and other.


Instruction:
Given the following Python 3 function, write the documentation
def as_dates(self):
        new_start = self.start.date() if self.start and isinstance(self.start, datetime.datetime) else self.start
        new_end = self.end.date() if self.end and isinstance(self.end, datetime.datetime) else self.end
        return DateRange(new_start, new_end, CLOSED_CLOSED)

Response: Create a new DateRange with the datetimes converted to dates.


Instruction:
Implement a function in Python 3 to
convert a DateRange into a MongoDb query string.

Response:
def mongo_query(self):
        """
        Convert a DateRange into a MongoDb query string. FIXME: Mongo can only handle
        datetimes in queries, so we should make this handle the case where start/end are
        datetime.date and extend accordingly (being careful about the interval logic).
        """
        comps = {OPEN_CLOSED: ('t', 'te'), OPEN_OPEN: ('t', 't'),
                 CLOSED_OPEN: ('te', 't'), CLOSED_CLOSED: ('te', 'te')}
        query = {}
        comp = comps[self.interval]
        if self.start:
            query['$g' + comp[0]] = self.start
        if self.end:
            query['$l' + comp[1]] = self.end
        return query


Instruction:
Create a Python 3 function to
return the upper and lower bounds along with operators that are needed to do an in range test.

Response:
def get_date_bounds(self):
        """
        Return the upper and lower bounds along
        with operators that are needed to do an 'in range' test.
        Useful for SQL commands.

        Returns
        -------
        tuple: (`str`, `date`, `str`, `date`)
                (date_gt, start, date_lt, end)
        e.g.:
                ('>=', start_date, '<', end_date)
        """
        start = end = None
        date_gt = '>='
        date_lt = '<='
        if self:
            if self.start:
                start = self.start
            if self.end:
                end = self.end
            if self.startopen:
                date_gt = '>'
            if self.endopen:
                date_lt = '<'

        return date_gt, start, date_lt, end


Instruction:
Write a Python 3 function that can
chunk the dataframe/series by dates Parameters ---------- df: pandas dataframe or series chunk_size: str any valid Pandas frequency string func: function func will be applied to each `chunk` generated by the chunker. This function CANNOT modify the date column of the dataframe! Returns ------- generator that produces tuples: (start date, end date, chunk_size, dataframe/series)

Response:
def to_chunks(self, df, chunk_size='D', func=None, **kwargs):
        """
        chunks the dataframe/series by dates

        Parameters
        ----------
        df: pandas dataframe or series
        chunk_size: str
            any valid Pandas frequency string
        func: function
            func will be applied to each `chunk` generated by the chunker.
            This function CANNOT modify the date column of the dataframe!

        Returns
        -------
        generator that produces tuples: (start date, end date,
                  chunk_size, dataframe/series)
        """
        if 'date' in df.index.names:
            dates = df.index.get_level_values('date')
            if not df.index.is_monotonic_increasing:
                df = df.sort_index()
        elif 'date' in df.columns:
            dates = pd.DatetimeIndex(df.date)
            if not dates.is_monotonic_increasing:
                # providing support for pandas 0.16.2 to 0.20.x
                # neither sort method exists in both
                try:
                    df = df.sort_values('date')
                except AttributeError:
                    df = df.sort(columns='date')
                dates = pd.DatetimeIndex(df.date)
        else:
            raise Exception("Data must be datetime indexed or have a column named 'date'")

        period_obj = dates.to_period(chunk_size)
        period_obj_reduced = period_obj.drop_duplicates()
        count = 0
        for _, g in df.groupby(period_obj._data):
            start = period_obj_reduced[count].start_time.to_pydatetime(warn=False)
            end = period_obj_reduced[count].end_time.to_pydatetime(warn=False)
            count += 1
            if func:
                yield start, end, chunk_size, func(g)
            else:
                yield start, end, chunk_size, g


Instruction:
Make a summary of the following Python 3 code
def to_mongo(self, range_obj):
        if isinstance(range_obj, (pd.DatetimeIndex, tuple)):
            range_obj = DateRange(range_obj[0], range_obj[-1])
        if range_obj.start and range_obj.end:
            return {'$and': [{START: {'$lte': range_obj.end}}, {END: {'$gte': range_obj.start}}]}
        elif range_obj.start:
            return {END: {'$gte': range_obj.start}}
        elif range_obj.end:
            return {START: {'$lte': range_obj.end}}
        else:
            return {}

Response: Converts a date range object to a MongoDB query that filters by the range


Instruction:
Can you write a function in Python 3 where it
filters the data by the specified date range.

Response:
def filter(self, data, range_obj):
        """
        ensures data is properly subset to the range in range_obj.
        (Depending on how the chunking is implemented, it might be possible
        to specify a chunk range that reads out more than the actual range
        eg: date range, chunked monthly. read out 2016-01-01 to 2016-01-02.
        This will read ALL of January 2016 but it should be subset to just
        the first two days)

        returns
        -------
        data, filtered by range_obj
        """
        if isinstance(range_obj, (pd.DatetimeIndex, tuple)):
            range_obj = DateRange(range_obj[0], range_obj[-1])

        range_obj = to_pandas_closed_closed(range_obj, add_tz=False)
        start = range_obj.start
        end = range_obj.end

        if 'date' in data.index.names:
            return data[start:end]
        elif 'date' in data.columns:
            if start and end:
                return data[(data.date >= start) & (data.date <= end)]
            elif start:
                return data[(data.date >= start)]
            elif end:
                return data[(data.date <= end)]
            else:
                return data
        else:
            return data


Instruction:
Here you have a function in Python 3, explain what it does
def exclude(self, data, range_obj):
        if isinstance(range_obj, (pd.DatetimeIndex, tuple)):
            range_obj = DateRange(range_obj[0], range_obj[-1])
        if 'date' in data.index.names:
            return data[(data.index.get_level_values('date') < range_obj.start) | (data.index.get_level_values('date') > range_obj.end)]
        elif 'date' in data.columns:
            return data[(data.date < range_obj.start) | (data.date > range_obj.end)]
        else:
            return data

Response: Removes data within the bounds of the range object


Instruction:
Explain what the following Python 3 code does
def enable_sharding(arctic, library_name, hashed=True, key='symbol'):
    c = arctic._conn
    lib = arctic[library_name]._arctic_lib
    dbname = lib._db.name
    library_name = lib.get_top_level_collection().name
    try:
        c.admin.command('enablesharding', dbname)
    except pymongo.errors.OperationFailure as e:
        if 'already enabled' not in str(e):
            raise
    if not hashed:
        logger.info("Range sharding '" + key + "' on: " + dbname + '.' + library_name)
        c.admin.command('shardCollection', dbname + '.' + library_name, key={key: 1})
    else:
        logger.info("Hash sharding '" + key + "' on: " + dbname + '.' + library_name)
        c.admin.command('shardCollection', dbname + '.' + library_name, key={key: 'hashed'})

Response: Enable sharding on a library.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def insert_one(self, document, **kwargs):
        self._arctic_lib.check_quota()
        return self._collection.insert_one(document, **kwargs)

Response: Insert one document into the collection.


Instruction:
Write a Python 3 script to
insert a list of documents into the collection.

Response:
def insert_many(self, documents, **kwargs):
        """
        See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.insert_many
        """
        self._arctic_lib.check_quota()
        return self._collection.insert_many(documents, **kwargs)


Instruction:
How would you explain what the following Python 3 function does
def update_one(self, filter, update, **kwargs):
        self._arctic_lib.check_quota()
        return self._collection.update_one(filter, update, **kwargs)

Response: Update one entry in the collection


Instruction:
Can you generate the documentation for the following Python 3 function
def update_many(self, filter, update, **kwargs):
        self._arctic_lib.check_quota()
        return self._collection.update_many(filter, update, **kwargs)

Response: Update many items in the collection


Instruction:
Can you create a Python 3 function that
replaces one entry in the collection with another.

Response:
def replace_one(self, filter, replacement, **kwargs):
        """
        See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.replace_one
        """
        self._arctic_lib.check_quota()
        return self._collection.replace_one(filter, replacement, **kwargs)


Instruction:
Implement a function in Python 3 to
find one and replace the object with replacement

Response:
def find_one_and_replace(self, filter, replacement, **kwargs):
        """
        See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.find_one_and_replace
        """
        self._arctic_lib.check_quota()
        return self._collection.find_one_and_replace(filter, replacement, **kwargs)


Instruction:
Can you tell what is the following Python 3 function doing
def find_one_and_update(self, filter, update, **kwargs):
        self._arctic_lib.check_quota()
        return self._collection.find_one_and_update(filter, update, **kwargs)

Response: Find one and update the object with the filter and kwargs


Instruction:
Write a Python 3 script to
bulk write to the collection.

Response:
def bulk_write(self, requests, **kwargs):
        """
        See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.bulk_write

        Warning: this is wrapped in mongo_retry, and is therefore potentially unsafe if the write you want to execute
        isn't idempotent.
        """
        self._arctic_lib.check_quota()
        return self._collection.bulk_write(requests, **kwargs)


Instruction:
Given the following Python 3 function, write the documentation
def count(self, filter, **kwargs):
        return mongo_count(self._collection, filter=filter, **kwargs)

Response: Count the number of items in the collection.


Instruction:
Can you generate the documentation for the following Python 3 function
def mktz(zone=None):
    if zone is None:
        zone = tzlocal.get_localzone().zone
    zone = six.u(zone)
    tz = dateutil.tz.gettz(zone)
    if not tz:
        raise TimezoneError('Timezone "%s" can not be read' % (zone))
    # Stash the zone name as an attribute (as pytz does)
    if not hasattr(tz, 'zone'):
        tz.zone = zone
        for p in dateutil.tz.TZPATHS:
            if zone.startswith(p):
                tz.zone = zone[len(p) + 1:]
                break
    return tz

Response: Create a new timezone object based on the zone name.


Instruction:
Write a Python 3 function that can
ensure that symbol(s) have contiguous segment ids Parameters ---------- library: arctic library symbol: None, str, list of str None: all symbols str: single symbol list: list of symbols Returns ------- list of str - Symbols 'fixed'

Response:
def segment_id_repair(library, symbol=None):
    """
    Ensure that symbol(s) have contiguous segment ids

    Parameters
    ----------
    library: arctic library
    symbol: None, str, list of str
        None: all symbols
        str: single symbol
        list: list of symbols

    Returns
    -------
    list of str - Symbols 'fixed'
    """
    ret = []

    if symbol is None:
        symbol = library.list_symbols()
    elif not isinstance(symbol, list):
        symbol = [symbol]

    by_segment = [(START, pymongo.ASCENDING),
                  (SEGMENT, pymongo.ASCENDING)]

    for sym in symbol:
        cursor = library._collection.find({SYMBOL: sym}, sort=by_segment)
        # group by chunk
        for _, segments in groupby(cursor, key=lambda x: (x[START], x[SYMBOL])):
            segments = list(segments)
            # if the start segment is not 0, we need to fix this symbol
            if segments[0][SEGMENT] == -1:
                # since the segment is part of the index, we have to clean up first
                library._collection.delete_many({SYMBOL: sym, START: segments[0][START]})
                # map each segment in the interval to the correct segment
                for index, seg in enumerate(segments):
                    seg[SEGMENT] = index
                library._collection.insert_many(segments)
                ret.append(sym)

    return ret


Instruction:
Here you have a function in Python 3, explain what it does
def multicolumn_store_with_uncompressed_write(mongo_server):
    mongo_server.api.drop_database('arctic_test')

    library_name = 'arctic_test.TEST'
    arctic = m.Arctic(mongo_host=mongo_server.api)
    arctic.initialize_library(library_name, m.VERSION_STORE, segment='month')

    db = mongo_server.api.arctic_test
    db.TEST.insert_many([
        {
            'parent': [bson.ObjectId('5ad0dc065c911d1188b512d8')],
            'data': bson.Binary(b'\x11\x00\x00\x002x\x01\x00\x01\x00\x80\x02\x00\x00\x00\x00\x00\x00\x00', 0),
            'symbol': 'pandas',
            'sha': bson.Binary(b'\xaa\\`\x0e\xc2D-\xc1_\xf7\xfd\x12\xfa\xd2\x17\x05`\x00\x98\xe2', 0),
            'compressed': True,
            '_id': bson.ObjectId('5ad0dc067934ecad404070be'),
            'segment': 0
        },
        {
            'parent': [bson.ObjectId('5ad0dc065c911d1188b512d8')],
            'data': bson.Binary(b'y\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00', 0),
            'symbol': 'pandas',
            'sha': bson.Binary(b'\xfe=WQ\xb5\xfdL\xb7\xcavd\x85o\x04]\x04\xdb\xa8]3', 0),
            'compressed': False,
            '_id': bson.ObjectId('5ad0dc077934ecad404070bf'),
            'segment': 1
        }
    ])
    db.TEST.ARCTIC.update_one({"_id": "ARCTIC_META"}, {"$set": {"_id": "ARCTIC_META", "TYPE": "VersionStore", "QUOTA": 10737418240}})
    db.TEST.version_nums.insert_one({'symbol': 'pandas', '_id': bson.ObjectId('5ad0dc067934ecad404070bd'), 'version': 2})
    db.TEST.versions.insert_many([
        {
            'append_count': 0,
            'dtype_metadata': {
                'index': ['index'],
                'columns': ["('a', 'a')", "('w', 'v')"]
            },
            'segment_count': 1,
            'dtype': '[(\'index\', \'S1\'), ("(\'a\', \'a\')", \'<i8\'), ("(\'w\', \'v\')", \'<i8\')]',
            'symbol': 'pandas',
            'up_to': 1,
            'metadata': None,
            'sha': bson.Binary(b'\xf2\x15h\x9d\x925\x95\xa5\x0e\x95J\xc4x\xfc\xfc\xd5\x80\xe0\x1d\xef', 0),
            'shape': [-1],
            'version': 1,
            'base_sha': bson.Binary(b'\xf2\x15h\x9d\x925\x95\xa5\x0e\x95J\xc4x\xfc\xfc\xd5\x80\xe0\x1d\xef', 0),
            '_id': bson.ObjectId('5ad0dc065c911d1188b512d8'),
            'type': 'pandasdf',
            'append_size': 0
        },
        {
            'append_count': 1,
            'dtype_metadata': {
                'index': ['index'],
                'columns': ["('a', 'a')", "('w', 'v')"]
            },
            'segment_count': 2,
            'sha': bson.Binary(b'1\x83[ZO\xec\x080D\x80f\xe4@\xe4\xd3\x94yG\xe2\x08', 0),
            'dtype': '[(\'index\', \'S1\'), ("(\'a\', \'a\')", \'<i8\'), ("(\'w\', \'v\')", \'<i8\')]',
            'symbol': 'pandas',
            'up_to': 2,
            'metadata': None,
            'base_version_id': bson.ObjectId('5ad0dc065c911d1188b512d8'),
            'shape': [-1],
            'version': 2,
            'base_sha': bson.Binary(b'\xf2\x15h\x9d\x925\x95\xa5\x0e\x95J\xc4x\xfc\xfc\xd5\x80\xe0\x1d\xef', 0),
            '_id': bson.ObjectId('5ad0dc075c911d1188b512d9'),
            'type': 'pandasdf',
            'append_size': 17
        }
    ])

    return {'symbol': 'pandas', 'store': arctic.get_library('arctic_test.TEST')}

Response: Stores the multicolumn of the current language in a single database.


Instruction:
How would you explain what the following Python 3 function does
def ndarray_store_with_uncompressed_write(mongo_server):
    mongo_server.api.drop_database('arctic_test')

    library_name = 'arctic_test.TEST'
    arctic = m.Arctic(mongo_host=mongo_server.api)
    arctic.initialize_library(library_name, m.VERSION_STORE, segment='month')

    db = mongo_server.api.arctic_test
    db.TEST.insert_many([
        {
            "_id": bson.ObjectId("5ad0742ca0949de6727cf994"),
            "segment": 0,
            "sha": bson.Binary(base64.b64decode("Fk+quqPVSDfaajYJkOAvnDyXtGQ="), 0),
            "symbol": "MYARR",
            "data": bson.Binary(base64.b64decode("CAAAAIAAAAAAAAAAAA=="), 0),
            "compressed": True,
            "parent": [bson.ObjectId("5ad0742c5c911d4d80ee2ea3")]
        },
        {
            "_id": bson.ObjectId("5ad0742ca0949de6727cf995"),
            "sha": bson.Binary(base64.b64decode("eqpp8VOJBttTz0j5H+QGtOQ+r44="), 0),
            "symbol": "MYARR",
            "segment": 1,
            "data": bson.Binary(base64.b64decode("AQAAAAAAAAA="), 0),
            "compressed": False,
            "parent": [bson.ObjectId("5ad0742c5c911d4d80ee2ea3")]
        }
    ])
    db.TEST.ARCTIC.update_one({"_id": "ARCTIC_META"}, {"$set": {"_id": "ARCTIC_META", "TYPE": "VersionStore", "QUOTA": 10737418240}})
    db.TEST.versions_nums.insert_one({"_id": bson.ObjectId("5ad0742ca0949de6727cf993"), "symbol": "MYARR", "version": 2})
    db.TEST.versions.insert_many([
        {
            "_id": bson.ObjectId("5ad0742c5c911d4d80ee2ea3"),
            "append_count": 0,
            "dtype_metadata": {},
            "segment_count": 1,
            "dtype": "[('abc', '<i8')]",
            "symbol": "MYARR",
            "up_to": 1,
            "append_size": 0,
            "sha": bson.Binary(base64.b64decode("Bf5AV1MWbxJVWefJrFWGVPEHx+k="), 0),
            "shape": [-1],
            "version": 1,
            "base_sha": bson.Binary(base64.b64decode("Bf5AV1MWbxJVWefJrFWGVPEHx+k="), 0),
            "type": "ndarray",
            "metadata": None
        },
        {
            "_id": bson.ObjectId("5ad0742c5c911d4d80ee2ea4"),
            "append_count": 1,
            "dtype_metadata": {},
            "segment_count": 2,
            "base_version_id": bson.ObjectId("5ad0742c5c911d4d80ee2ea3"),
            "dtype": "[('abc', '<i8')]",
            "symbol": "MYARR",
            "up_to": 2,
            "append_size": 8,
            "sha": bson.Binary(base64.b64decode("Ax7oBxVFw1/9wKog2gfOLjbOVD8="), 0),
            "shape": [-1],
            "version": 2,
            "base_sha": bson.Binary(base64.b64decode("Bf5AV1MWbxJVWefJrFWGVPEHx+k="), 0),
            "type": "ndarray",
            "metadata": None
        }
    ])

    return {'symbol': 'MYARR', 'store': arctic.get_library('arctic_test.TEST')}

Response: Stores the array of objects in the database with uncompressed writes.


Instruction:
Given the following Python 3 function, write the documentation
def list_symbols(self, all_symbols=False, snapshot=None, regex=None, **kwargs):
        query = {}
        if regex is not None:
            query['symbol'] = {'$regex': regex}
        if kwargs:
            for k, v in six.iteritems(kwargs):
                # TODO: this doesn't work as expected as it ignores the versions with metadata.deleted set
                #       as a result it will return symbols with matching metadata which have been deleted
                #       Maybe better add a match step in the pipeline instead of making it part of the query
                query['metadata.' + k] = v
        if snapshot is not None:
            try:
                query['parent'] = self._snapshots.find_one({'name': snapshot})['_id']
            except TypeError:
                raise NoDataFoundException('No snapshot %s in library %s' % (snapshot, self._arctic_lib.get_name()))
        elif all_symbols:
            return self._versions.find(query).distinct('symbol')

        # Return just the symbols which aren't deleted in the 'trunk' of this library
        pipeline = []
        if query:
            # Match based on user criteria first
            pipeline.append({'$match': query})
        pipeline.extend([
            # version_custom value is: 2*version + (0 if deleted else 1)
            # This is used to optimize aggregation query:
            #  - avoid sorting
            #  - be able to rely on the latest version (max) for the deleted status
            #
            # Be aware of that if you don't use custom sort or if use a sort before $group which utilizes
            # exactly an existing index, the $group will do best effort to utilize this index:
            #  - https://jira.mongodb.org/browse/SERVER-4507
            {'$group': {
                '_id': '$symbol',
                'version_custom': {
                    '$max': {
                        '$add': [
                            {'$multiply': ['$version', 2]},
                            {'$cond': [{'$eq': ['$metadata.deleted', True]}, 1, 0]}
                        ]
                    }
                },
            }},
            # Don't include symbols which are part of some snapshot, but really deleted...
            {'$match': {'version_custom': {'$mod': [2, 0]}}}
        ])

        # We may hit the group memory limit (100MB), so use allowDiskUse to circumvent this
        #  - https://docs.mongodb.com/manual/reference/operator/aggregation/group/#group-memory-limit
        return sorted([x['_id'] for x in self._versions.aggregate(pipeline, allowDiskUse=True)])

Response: Returns a list of all available symbols in this library.


Instruction:
How would you explain what the following Python 3 function does
def has_symbol(self, symbol, as_of=None):
        try:
            # Always use the primary for has_symbol, it's safer
            self._read_metadata(symbol, as_of=as_of, read_preference=ReadPreference.PRIMARY)
            return True
        except NoDataFoundException:
            return False

Response: Returns True if the symbol exists in this library AND the version number of the version number is greater than or equal to the requested point in time.


Instruction:
Given the following Python 3 function, write the documentation
def read_audit_log(self, symbol=None, message=None):
        query = {}
        if symbol:
            if isinstance(symbol, six.string_types):
                query['symbol'] = {'$regex': symbol}
            else:
                query['symbol'] = {'$in': list(symbol)}

        if message is not None:
            query['message'] = message

        def _pop_id(x):
            x.pop('_id')
            return x

        return [_pop_id(x) for x in self._audit.find(query, sort=[('_id', -1)])]

Response: Reads the audit log associated with a given symbol.


Instruction:
Implement a Python 3 function for
returning a list of versions for the specified symbol and snapshot.

Response:
def list_versions(self, symbol=None, snapshot=None, latest_only=False):
        """
        Return a list of versions filtered by the passed in parameters.

        Parameters
        ----------
        symbol : `str`
            Symbol to return versions for.  If None returns versions across all
            symbols in the library.
        snapshot : `str`
            Return the versions contained in the named snapshot
        latest_only : `bool`
            Only include the latest version for a specific symbol

        Returns
        -------
        List of dictionaries describing the discovered versions in the library
        """
        if symbol is None:
            symbols = self.list_symbols(snapshot=snapshot)
        else:
            symbols = [symbol]

        query = {}

        if snapshot is not None:
            try:
                query['parent'] = self._snapshots.find_one({'name': snapshot})['_id']
            except TypeError:
                raise NoDataFoundException('No snapshot %s in library %s' % (snapshot, self._arctic_lib.get_name()))

        versions = []
        snapshots = {ss.get('_id'): ss.get('name') for ss in self._snapshots.find()}
        for symbol in symbols:
            query['symbol'] = symbol
            seen_symbols = set()
            for version in self._versions.find(query, projection=['symbol', 'version', 'parent', 'metadata.deleted'], sort=[('version', -1)]):
                if latest_only and version['symbol'] in seen_symbols:
                    continue
                seen_symbols.add(version['symbol'])
                meta = version.get('metadata')
                versions.append({'symbol': version['symbol'], 'version': version['version'],
                                 'deleted': meta.get('deleted', False) if meta else False,
                                 # We return offset-aware datetimes in Local Time.
                                 'date': ms_to_datetime(datetime_to_ms(version['_id'].generation_time)),
                                 'snapshots': [snapshots[s] for s in version.get('parent', []) if s in snapshots]})
        return versions


Instruction:
How would you implement a function in Python 3 that
reads the data for the named symbol from the version store.

Response:
def read(self, symbol, as_of=None, date_range=None, from_version=None, allow_secondary=None, **kwargs):
        """
        Read data for the named symbol.  Returns a VersionedItem object with
        a data and metdata element (as passed into write).

        Parameters
        ----------
        symbol : `str`
            symbol name for the item
        as_of : `str` or `int` or `datetime.datetime`
            Return the data as it was as_of the point in time.
            `int` : specific version number
            `str` : snapshot name which contains the version
            `datetime.datetime` : the version of the data that existed as_of the requested point in time
        date_range: `arctic.date.DateRange`
            DateRange to read data for.  Applies to Pandas data, with a DateTime index
            returns only the part of the data that falls in the DateRange.
        allow_secondary : `bool` or `None`
            Override the default behavior for allowing reads from secondary members of a cluster:
            `None` : use the settings from the top-level `Arctic` object used to query this version store.
            `True` : allow reads from secondary members
            `False` : only allow reads from primary members

        Returns
        -------
        VersionedItem namedtuple which contains a .data and .metadata element
        """
        try:
            read_preference = self._read_preference(allow_secondary)
            _version = self._read_metadata(symbol, as_of=as_of, read_preference=read_preference)
            return self._do_read(symbol, _version, from_version,
                                 date_range=date_range, read_preference=read_preference, **kwargs)
        except (OperationFailure, AutoReconnect) as e:
            # Log the exception so we know how often this is happening
            log_exception('read', e, 1)
            # If we've failed to read from the secondary, then it's possible the
            # secondary has lagged.  In this case direct the query to the primary.
            _version = mongo_retry(self._read_metadata)(symbol, as_of=as_of,
                                                        read_preference=ReadPreference.PRIMARY)
            return self._do_read_retry(symbol, _version, from_version,
                                       date_range=date_range,
                                       read_preference=ReadPreference.PRIMARY,
                                       **kwargs)
        except Exception as e:
            log_exception('read', e, 1)
            raise


Instruction:
Can you tell what is the following Python 3 function doing
def get_info(self, symbol, as_of=None):
        version = self._read_metadata(symbol, as_of=as_of, read_preference=None)
        handler = self._read_handler(version, symbol)
        if handler and hasattr(handler, 'get_info'):
            return handler.get_info(version)
        return {}

Response: Reads and returns the information about the item with the given symbol.


Instruction:
Write a Python 3 function that can
return the numerical representation of the arctic version used to write the last version for the given symbol.

Response:
def get_arctic_version(self, symbol, as_of=None):
        """
        Return the numerical representation of the arctic version used to write the last (or as_of) version for
        the given symbol.

        Parameters
        ----------
        symbol : `str`
            symbol name for the item
        as_of : `str` or int or `datetime.datetime`
            Return the data as it was as_of the point in time.
            `int` : specific version number
            `str` : snapshot name which contains the version
            `datetime.datetime` : the version of the data that existed as_of the requested point in time

        Returns
        -------
        arctic_version : int
            The numerical representation of Arctic version, used to create the specified symbol version
        """
        return self._read_metadata(symbol, as_of=as_of).get('arctic_version', 0)


Instruction:
Explain what the following Python 3 code does
def read_metadata(self, symbol, as_of=None, allow_secondary=None):
        _version = self._read_metadata(symbol, as_of=as_of, read_preference=self._read_preference(allow_secondary))
        return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=_version['version'],
                             metadata=_version.pop('metadata', None), data=None,
                             host=self._arctic_lib.arctic.mongo_host)

Response: Reads the metadata for a specific item.


Instruction:
How would you explain what the following Python 3 function does
def append(self, symbol, data, metadata=None, prune_previous_version=True, upsert=True, **kwargs):
        self._arctic_lib.check_quota()
        version = {'_id': bson.ObjectId()}
        version['arctic_version'] = ARCTIC_VERSION_NUMERICAL
        version['symbol'] = symbol
        spec = {'symbol': symbol}
        previous_version = self._versions.find_one(spec,
                                                   sort=[('version', pymongo.DESCENDING)])

        if len(data) == 0 and previous_version is not None:
            return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=previous_version['version'],
                                 metadata=version.pop('metadata', None), data=None,
                                 host=self._arctic_lib.arctic.mongo_host)

        if upsert and previous_version is None:
            return self.write(symbol=symbol, data=data, prune_previous_version=prune_previous_version, metadata=metadata)

        assert previous_version is not None
        dirty_append = False

        # Take a version number for this append.
        # If the version numbers of this and the previous version aren't sequential,
        # then we've either had a failed append in the past,
        # we're in the midst of a concurrent update, we've deleted a version in-between
        # or are somehow 'forking' history
        next_ver = self._version_nums.find_one_and_update({'symbol': symbol, },
                                                          {'$inc': {'version': 1}},
                                                          upsert=False, new=True)['version']

        # This is a very important check, do not remove/modify as it is a guard-dog preventing potential data corruption
        if next_ver != previous_version['version'] + 1:
            dirty_append = True
            logger.debug('''version_nums is out of sync with previous version document.
            This probably means that either a version document write has previously failed, or the previous version has been deleted.''')

        # if the symbol has previously been deleted then overwrite
        previous_metadata = previous_version.get('metadata', None)
        if upsert and previous_metadata is not None and previous_metadata.get('deleted', False) is True:
            return self.write(symbol=symbol, data=data, prune_previous_version=prune_previous_version,
                              metadata=metadata)

        handler = self._read_handler(previous_version, symbol)

        if metadata is not None:
            version['metadata'] = metadata
        elif 'metadata' in previous_version:
            version['metadata'] = previous_version['metadata']

        if handler and hasattr(handler, 'append') and callable(handler.append):
            handler.append(self._arctic_lib, version, symbol, data,
                           previous_version, dirty_append=dirty_append, **kwargs)
        else:
            raise Exception("Append not implemented for handler %s" % handler)

        if prune_previous_version and previous_version:
            # Does not allow prune to remove the base of the new version
            self._prune_previous_versions(symbol, keep_version=version.get('base_version_id'),
                                          new_version_shas=version.get(FW_POINTERS_REFS_KEY))

        # Insert the new version into the version DB
        version['version'] = next_ver
        self._insert_version(version)

        return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=version['version'],
                             metadata=version.pop('metadata', None), data=None,
                             host=self._arctic_lib.arctic.mongo_host)

Response: Append data under the specified symbol name to this library.


Instruction:
Given the following Python 3 function, write the documentation
def write(self, symbol, data, metadata=None, prune_previous_version=True, **kwargs):
        self._arctic_lib.check_quota()
        version = {'_id': bson.ObjectId()}
        version['arctic_version'] = ARCTIC_VERSION_NUMERICAL
        version['symbol'] = symbol
        version['version'] = self._version_nums.find_one_and_update({'symbol': symbol},
                                                                    {'$inc': {'version': 1}},
                                                                    upsert=True, new=True)['version']
        version['metadata'] = metadata

        previous_version = self._versions.find_one({'symbol': symbol, 'version': {'$lt': version['version']}},
                                                   sort=[('version', pymongo.DESCENDING)])

        handler = self._write_handler(version, symbol, data, **kwargs)
        handler.write(self._arctic_lib, version, symbol, data, previous_version, **kwargs)

        if prune_previous_version and previous_version:
            self._prune_previous_versions(symbol, new_version_shas=version.get(FW_POINTERS_REFS_KEY))

        # Insert the new version into the version DB
        self._insert_version(version)

        logger.debug('Finished writing versions for %s', symbol)

        return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=version['version'],
                             metadata=version.pop('metadata', None), data=None,
                             host=self._arctic_lib.arctic.mongo_host)

Response: Writes the data under the specified symbol to this library.


Instruction:
Can you generate the documentation for the following Python 3 function
def write_metadata(self, symbol, metadata, prune_previous_version=True, **kwargs):
        # Make a normal write with empty data and supplied metadata if symbol does not exist
        try:
            previous_version = self._read_metadata(symbol)
        except NoDataFoundException:
            return self.write(symbol, data=None, metadata=metadata,
                              prune_previous_version=prune_previous_version, **kwargs)

        # Reaching here means that and/or metadata exist and we are set to update the metadata
        new_version_num = self._version_nums.find_one_and_update({'symbol': symbol},
                                                                 {'$inc': {'version': 1}},
                                                                 upsert=True, new=True)['version']

        # Populate the new version entry, preserving existing data, and updating with the supplied metadata
        version = {k: previous_version[k] for k in previous_version.keys() if k != 'parent'}   # don't copy snapshots
        version['_id'] = bson.ObjectId()
        version['version'] = new_version_num
        version['metadata'] = metadata
        version['base_version_id'] = previous_version.get('base_version_id', previous_version['_id'])

        return self._add_new_version_using_reference(symbol, version, previous_version, prune_previous_version)

Response: Writes the metadata for the specified symbol to this library.


Instruction:
Given the following Python 3 function, write the documentation
def restore_version(self, symbol, as_of, prune_previous_version=True):
        # TODO: This operation is tricky as it may create history branches and lead to corrupted symbols.
        #       To avoid this we do concat_rewrite (see Issue #579)
        #       Investigate how this can be optimized and maintain safety (i.e. avoid read/write with serialization
        #       and compression costs, but instead:
        #       clone segments (server-side?) / crate new (base) version / update segments' parent).

        version_to_restore = self._read_metadata(symbol, as_of=as_of)

        # At this point it is guaranteed that the as_of version exists and doesn't have the symbol marked as deleted.
        # If we try to restore the last version, do nothing (No-Op) and return the associated VesionedItem.
        if self._last_version_seqnum(symbol) == version_to_restore['version']:
            return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(),
                                 version=version_to_restore['version'],
                                 host=self._arctic_lib.arctic.mongo_host,
                                 metadata=version_to_restore.pop('metadata', None), data=None)

        # Read the existing data from as_of
        item = self.read(symbol, as_of=as_of)

        # Write back, creating a new base version
        new_item = self.write(symbol,
                              data=item.data, metadata=item.metadata, prune_previous_version=prune_previous_version)
        return new_item

Response: Restores the specified symbol s data and metadata to the state of a given version.


Instruction:
Make a summary of the following Python 3 code
def _find_prunable_version_ids(self, symbol, keep_mins):
        read_preference = ReadPreference.SECONDARY_PREFERRED if keep_mins > 0 else ReadPreference.PRIMARY
        versions = self._versions.with_options(read_preference=read_preference)
        query = {'symbol': symbol,
                 # Not snapshotted
                 '$or': [{'parent': {'$exists': False}}, {'parent': []}],
                 # At least 'keep_mins' old
                 '_id': {'$lt': bson.ObjectId.from_datetime(dt.utcnow()
                                                            # Add one second as the ObjectId
                                                            # str has random fuzz
                                                            + timedelta(seconds=1)
                                                            - timedelta(minutes=keep_mins)
                                                            )
                         }
                 }
        cursor = versions.find(query,
                               # Using version number here instead of _id as there's a very unlikely case
                               # where the versions are created on different hosts or processes at exactly
                               # the same time.
                               sort=[('version', pymongo.DESCENDING)],
                               # Guarantees at least one version is kept
                               skip=1,
                               projection={'_id': 1, FW_POINTERS_REFS_KEY: 1, FW_POINTERS_CONFIG_KEY: 1},
                               )
        return {v['_id']: ([bson.binary.Binary(x) for x in v.get(FW_POINTERS_REFS_KEY, [])], get_fwptr_config(v))
                for v in cursor}

Response: Find all non - snapshotted versions of a symbol that are older than keep_mins.


Instruction:
Can you tell what is the following Python 3 function doing
def _find_base_version_ids(self, symbol, version_ids):
        cursor = self._versions.find({'symbol': symbol,
                                      '_id': {'$nin': version_ids},
                                      'base_version_id': {'$exists': True},
                                      },
                                     projection={'base_version_id': 1})
        return [version["base_version_id"] for version in cursor]

Response: Find all base_version_ids for a symbol that are not bases of version_ids


Instruction:
Make a summary of the following Python 3 code
def _prune_previous_versions(self, symbol, keep_mins=120, keep_version=None, new_version_shas=None):
        new_version_shas = new_version_shas if new_version_shas else []
        prunable_ids_to_shas = self._find_prunable_version_ids(symbol, keep_mins)
        prunable_ids = list(prunable_ids_to_shas.keys())
        if keep_version is not None:
            try:
                prunable_ids.remove(keep_version)
            except ValueError:
                pass
        if not prunable_ids:
            return

        base_version_ids = self._find_base_version_ids(symbol, prunable_ids)
        version_ids = list(set(prunable_ids) - set(base_version_ids))
        if not version_ids:
            return

        # Delete the version documents
        mongo_retry(self._versions.delete_many)({'_id': {'$in': version_ids}})

        prunable_ids_to_shas = {k: prunable_ids_to_shas[k] for k in version_ids}

        # The new version has not been written yet, so make sure that any SHAs pointed by it are preserved
        shas_to_delete = [sha for v in prunable_ids_to_shas.values() for sha in v[0] if sha not in new_version_shas]

        # Cleanup any chunks
        mongo_retry(cleanup)(self._arctic_lib, symbol, version_ids, self._versions,
                             shas_to_delete=shas_to_delete,
                             pointers_cfgs=[v[1] for v in prunable_ids_to_shas.values()])

Response: Prune versions not pointed at by snapshots which are at least keep_mins old.