Instruction:
Can you generate a brief explanation for the following Python 3 code
def _rebuildPartitionIdMap(self, partitionIdList):
    self._partitionIdMap = {}
    for row, partitionId in enumerate(partitionIdList):
      indices = self._partitionIdMap.get(partitionId, [])
      indices.append(row)
      self._partitionIdMap[partitionId] = indices

Response: Rebuilds the partition Id map using the given list of partition ids.


Instruction:
Write a Python 3 function that can
calculate the distances from inputPattern to all stored patterns.

Response:
def _calcDistance(self, inputPattern, distanceNorm=None):
    """Calculate the distances from inputPattern to all stored patterns. All
    distances are between 0.0 and 1.0

    :param inputPattern The pattern from which distances to all other patterns
        are calculated

    :param distanceNorm Degree of the distance norm
    """
    if distanceNorm is None:
      distanceNorm = self.distanceNorm

    # Sparse memory
    if self.useSparseMemory:
      if self._protoSizes is None:
        self._protoSizes = self._Memory.rowSums()
      overlapsWithProtos = self._Memory.rightVecSumAtNZ(inputPattern)
      inputPatternSum = inputPattern.sum()

      if self.distanceMethod == "rawOverlap":
        dist = inputPattern.sum() - overlapsWithProtos
      elif self.distanceMethod == "pctOverlapOfInput":
        dist = inputPatternSum - overlapsWithProtos
        if inputPatternSum > 0:
          dist /= inputPatternSum
      elif self.distanceMethod == "pctOverlapOfProto":
        overlapsWithProtos /= self._protoSizes
        dist = 1.0 - overlapsWithProtos
      elif self.distanceMethod == "pctOverlapOfLarger":
        maxVal = numpy.maximum(self._protoSizes, inputPatternSum)
        if maxVal.all() > 0:
          overlapsWithProtos /= maxVal
        dist = 1.0 - overlapsWithProtos
      elif self.distanceMethod == "norm":
        dist = self._Memory.vecLpDist(self.distanceNorm, inputPattern)
        distMax = dist.max()
        if distMax > 0:
          dist /= distMax
      else:
        raise RuntimeError("Unimplemented distance method %s" %
          self.distanceMethod)

    # Dense memory
    else:
      if self.distanceMethod == "norm":
        dist = numpy.power(numpy.abs(self._M - inputPattern), self.distanceNorm)
        dist = dist.sum(1)
        dist = numpy.power(dist, 1.0/self.distanceNorm)
        dist /= dist.max()
      else:
        raise RuntimeError ("Not implemented yet for dense storage....")

    return dist


Instruction:
Write a Python 3 script to
compute the distances from inputPattern to all stored patterns.

Response:
def _getDistances(self, inputPattern, partitionId=None):
    """Return the distances from inputPattern to all stored patterns.

    :param inputPattern The pattern from which distances to all other patterns
        are returned

    :param partitionId If provided, ignore all training vectors with this
        partitionId.
    """
    if not self._finishedLearning:
      self.finishLearning()
      self._finishedLearning = True

    if self._vt is not None and len(self._vt) > 0:
      inputPattern = numpy.dot(self._vt, inputPattern - self._mean)

    sparseInput = self._sparsifyVector(inputPattern)

    # Compute distances
    dist = self._calcDistance(sparseInput)
    # Invalidate results where category is -1
    if self._specificIndexTraining:
      dist[numpy.array(self._categoryList) == -1] = numpy.inf

    # Ignore vectors with this partition id by setting their distances to inf
    if partitionId is not None:
      dist[self._partitionIdMap.get(partitionId, [])] = numpy.inf

    return dist


Instruction:
Can you tell what is the following Python 3 function doing
def computeSVD(self, numSVDSamples=0, finalize=True):
    if numSVDSamples == 0:
      numSVDSamples = self._numPatterns

    if not self.useSparseMemory:
      self._a = self._Memory[:self._numPatterns]
    else:
      self._a = self._Memory.toDense()[:self._numPatterns]

    self._mean = numpy.mean(self._a, axis=0)
    self._a -= self._mean
    u,self._s,self._vt = numpy.linalg.svd(self._a[:numSVDSamples])

    if finalize:
      self._finalizeSVD()

    return self._s

Response: Compute the singular value decomposition of the input patterns.


Instruction:
Make a summary of the following Python 3 code
def getAdaptiveSVDDims(self, singularValues, fractionOfMax=0.001):
    v = singularValues/singularValues[0]
    idx = numpy.where(v<fractionOfMax)[0]
    if len(idx):
      print "Number of PCA dimensions chosen: ", idx[0], "out of ", len(v)
      return idx[0]
    else:
      print "Number of PCA dimensions chosen: ", len(v)-1, "out of ", len(v)
      return len(v)-1

Response: Compute the number of eigenvectors to keep.


Instruction:
Given the following Python 3 function, write the documentation
def _finalizeSVD(self, numSVDDims=None):
    if numSVDDims is not None:
      self.numSVDDims = numSVDDims

    if self.numSVDDims=="adaptive":
      if self.fractionOfMax is not None:
          self.numSVDDims = self.getAdaptiveSVDDims(self._s, self.fractionOfMax)
      else:
          self.numSVDDims = self.getAdaptiveSVDDims(self._s)

    if self._vt.shape[0] < self.numSVDDims:
      print "******************************************************************"
      print ("Warning: The requested number of PCA dimensions is more than "
             "the number of pattern dimensions.")
      print "Setting numSVDDims = ", self._vt.shape[0]
      print "******************************************************************"
      self.numSVDDims = self._vt.shape[0]

    self._vt = self._vt[:self.numSVDDims]

    # Added when svd is not able to decompose vectors - uses raw spare vectors
    if len(self._vt) == 0:
      return

    self._Memory = numpy.zeros((self._numPatterns,self.numSVDDims))
    self._M = self._Memory
    self.useSparseMemory = False

    for i in range(self._numPatterns):
      self._Memory[i] = numpy.dot(self._vt, self._a[i])

    self._a = None

Response: Called by finalizeLearning. This will finalize the svd and compute the eigenvectors.


Instruction:
Given the following Python 3 function, write the documentation
def remapCategories(self, mapping):
    categoryArray = numpy.array(self._categoryList)
    newCategoryArray = numpy.zeros(categoryArray.shape[0])
    newCategoryArray.fill(-1)
    for i in xrange(len(mapping)):
      newCategoryArray[categoryArray==i] = mapping[i]
    self._categoryList = list(newCategoryArray)

Response: Change the category indices in the image sensor categoryInfo array.


Instruction:
Implement a function in Python 3 to
change the category associated with this vector.

Response:
def setCategoryOfVectors(self, vectorIndices, categoryIndices):
    """Change the category associated with this vector(s).

    Used by the Network Builder to move vectors between categories, to enable
    categories, and to invalidate vectors by setting the category to -1.

    :param vectorIndices: Single index or list of indices

    :param categoryIndices: Single index or list of indices. Can also be a
        single index when vectorIndices is a list, in which case the same
        category will be used for all vectors
    """
    if not hasattr(vectorIndices, "__iter__"):
      vectorIndices = [vectorIndices]
      categoryIndices = [categoryIndices]
    elif not hasattr(categoryIndices, "__iter__"):
      categoryIndices = [categoryIndices] * len(vectorIndices)

    for i in xrange(len(vectorIndices)):
      vectorIndex = vectorIndices[i]
      categoryIndex = categoryIndices[i]

      # Out-of-bounds is not an error, because the KNN may not have seen the
      # vector yet
      if vectorIndex < len(self._categoryList):
        self._categoryList[vectorIndex] = categoryIndex


Instruction:
Can you create a Python 3 function that
returns the next record in the list of records to encode.

Response:
def getNextRecord(self):
    """
    Get the next record to encode. Includes getting a record from the
    `dataSource` and applying filters. If the filters request more data from the
    `dataSource` continue to get data from the `dataSource` until all filters
    are satisfied. This method is separate from :meth:`.RecordSensor.compute` so that we can
    use a standalone :class:`.RecordSensor` to get filtered data.
    """

    allFiltersHaveEnoughData = False
    while not allFiltersHaveEnoughData:
      # Get the data from the dataSource
      data = self.dataSource.getNextRecordDict()

      if not data:
        raise StopIteration("Datasource has no more data")

      # temporary check
      if "_reset" not in data:
        data["_reset"] = 0
      if "_sequenceId" not in data:
        data["_sequenceId"] = 0
      if "_category" not in data:
        data["_category"] = [None]

      data, allFiltersHaveEnoughData = self.applyFilters(data)

    self.lastRecord = data

    return data


Instruction:
Create a Python 3 function to
apply the pre - encoding filters to the data and returns the data and all filters have enough data.

Response:
def applyFilters(self, data):
    """
    Apply pre-encoding filters. These filters may modify or add data. If a
    filter needs another record (e.g. a delta filter) it will request another
    record by returning False and the current record will be skipped (but will
    still be given to all filters).

    We have to be very careful about resets. A filter may add a reset,
    but other filters should not see the added reset, each filter sees
    the original reset value, and we keep track of whether any filter
    adds a reset.

    :param data: (dict) The data that will be processed by the filter.
    :returns: (tuple) with the data processed by the filter and a boolean to
              know whether or not the filter needs mode data.
    """

    if self.verbosity > 0:
      print "RecordSensor got data: %s" % data

    allFiltersHaveEnoughData = True
    if len(self.preEncodingFilters) > 0:
      originalReset = data['_reset']
      actualReset = originalReset
      for f in self.preEncodingFilters:
        # if filter needs more data, it returns False
        filterHasEnoughData = f.process(data)
        allFiltersHaveEnoughData = (allFiltersHaveEnoughData
                                    and filterHasEnoughData)
        actualReset = actualReset or data['_reset']
        data['_reset'] = originalReset
      data['_reset'] = actualReset

    return data, allFiltersHaveEnoughData


Instruction:
Here you have a function in Python 3, explain what it does
def populateCategoriesOut(self, categories, output):
    if categories[0] is None:
      # The record has no entry in category field.
      output[:] = -1
    else:
      # Populate category output array by looping over the smaller of the
      # output array (size specified by numCategories) and the record's number
      # of categories.
      for i, cat in enumerate(categories[:len(output)]):
        output[i] = cat
      output[len(categories):] = -1

Response: Populate the output array with the category indices.


Instruction:
Make a summary of the following Python 3 code
def compute(self, inputs, outputs):
    if not self.topDownMode:
      data = self.getNextRecord()

      # The private keys in data are standard of RecordStreamIface objects. Any
      # add'l keys are column headers from the data source.
      reset = data["_reset"]
      sequenceId = data["_sequenceId"]
      categories = data["_category"]

      # Encode the processed records; populate outputs["dataOut"] in place
      self.encoder.encodeIntoArray(data, outputs["dataOut"])

      # If there is a field to predict, set bucketIdxOut and actValueOut.
      # There is a special case where a predicted field might be a vector, as in
      # the CoordinateEncoder. Since this encoder does not provide bucket
      # indices for prediction, we will ignore it.
      if self.predictedField is not None and self.predictedField != "vector":
        allEncoders = list(self.encoder.encoders)
        if self.disabledEncoder is not None:
          allEncoders.extend(self.disabledEncoder.encoders)
        encoders = [e for e in allEncoders
                    if e[0] == self.predictedField]
        if len(encoders) == 0:
          raise ValueError("There is no encoder for set for the predicted "
                           "field: %s" % self.predictedField)
        # TODO: Figure out why there are sometimes multiple encoders with the
        # same name.
        #elif len(encoders) > 1:
        #  raise ValueError("There cant' be more than 1 encoder for the "
        #                   "predicted field: %s" % self.predictedField)
        else:
          encoder = encoders[0][1]

        actualValue = data[self.predictedField]
        outputs["bucketIdxOut"][:] = encoder.getBucketIndices(actualValue)
        if isinstance(actualValue, str):
          outputs["actValueOut"][:] = encoder.getBucketIndices(actualValue)
        else:
          outputs["actValueOut"][:] = actualValue

      # Write out the scalar values obtained from they data source.
      outputs["sourceOut"][:] = self.encoder.getScalars(data)
      self._outputValues["sourceOut"] = self.encoder.getEncodedValues(data)

      # -----------------------------------------------------------------------
      # Get the encoded bit arrays for each field
      encoders = self.encoder.getEncoderList()
      prevOffset = 0
      sourceEncodings = []
      bitData = outputs["dataOut"]
      for encoder in encoders:
        nextOffset = prevOffset + encoder.getWidth()
        sourceEncodings.append(bitData[prevOffset:nextOffset])
        prevOffset = nextOffset
      self._outputValues['sourceEncodings'] = sourceEncodings

      # Execute post-encoding filters, if any
      for filter in self.postEncodingFilters:
        filter.process(encoder=self.encoder, data=outputs['dataOut'])

      # Populate the output numpy arrays; must assign by index.
      outputs['resetOut'][0] = reset
      outputs['sequenceIdOut'][0] = sequenceId
      self.populateCategoriesOut(categories, outputs['categoryOut'])

      # ------------------------------------------------------------------------
      # Verbose print?
      if self.verbosity >= 1:
        if self._iterNum == 0:
          self.encoder.pprintHeader(prefix="sensor:")
        if reset:
          print "RESET - sequenceID:%d" % sequenceId
        if self.verbosity >= 2:
          print

      # If verbosity >=2, print the record fields
      if self.verbosity >= 1:
        self.encoder.pprint(outputs["dataOut"], prefix="%7d:" % (self._iterNum))
        scalarValues = self.encoder.getScalars(data)
        nz = outputs["dataOut"].nonzero()[0]
        print "     nz: (%d)" % (len(nz)), nz
        print "  encIn:", self.encoder.scalarsToStr(scalarValues)
      if self.verbosity >= 2:
        # if hasattr(data, 'header'):
        #  header = data.header()
        # else:
        #  header = '     '.join(self.dataSource.names)
        # print "        ", header
        print "   data:", str(data)
      if self.verbosity >= 3:
        decoded = self.encoder.decode(outputs["dataOut"])
        print "decoded:", self.encoder.decodedToStr(decoded)

      self._iterNum += 1

    else:

      # ========================================================================
      # Spatial
      # ========================================================================
      # This is the top down compute in sensor

      # We get the spatial pooler's topDownOut as spatialTopDownIn
      spatialTopDownIn = inputs['spatialTopDownIn']
      spatialTopDownOut = self.encoder.topDownCompute(spatialTopDownIn)

      # -----------------------------------------------------------------------
      # Split topDownOutput into seperate outputs
      values = [elem.value for elem in spatialTopDownOut]
      scalars = [elem.scalar for elem in spatialTopDownOut]
      encodings = [elem.encoding for elem in spatialTopDownOut]
      self._outputValues['spatialTopDownOut'] = values
      outputs['spatialTopDownOut'][:] = numpy.array(scalars)
      self._outputValues['spatialTopDownEncodings'] = encodings

      # ========================================================================
      # Temporal
      # ========================================================================

      ## TODO: Add temporal top-down loop
      # We get the temporal memory's topDownOut passed through the spatial
      # pooler as temporalTopDownIn
      temporalTopDownIn = inputs['temporalTopDownIn']
      temporalTopDownOut = self.encoder.topDownCompute(temporalTopDownIn)

      # -----------------------------------------------------------------------
      # Split topDownOutput into separate outputs

      values = [elem.value for elem in temporalTopDownOut]
      scalars = [elem.scalar for elem in temporalTopDownOut]
      encodings = [elem.encoding for elem in temporalTopDownOut]
      self._outputValues['temporalTopDownOut'] = values
      outputs['temporalTopDownOut'][:] = numpy.array(scalars)
      self._outputValues['temporalTopDownEncodings'] = encodings

      assert len(spatialTopDownOut) == len(temporalTopDownOut), (
        "Error: spatialTopDownOut and temporalTopDownOut should be the same "
        "size")

Response: Compute the next record from the dataSource and encode it.


Instruction:
Can you generate the documentation for the following Python 3 function
def _convertNonNumericData(self, spatialOutput, temporalOutput, output):
    encoders = self.encoder.getEncoderList()
    types = self.encoder.getDecoderOutputFieldTypes()
    for i, (encoder, type) in enumerate(zip(encoders, types)):
      spatialData = spatialOutput[i]
      temporalData = temporalOutput[i]

      if type != FieldMetaType.integer and type != FieldMetaType.float:
        # TODO: Make sure that this doesn't modify any state
        spatialData = encoder.getScalars(spatialData)[0]
        temporalData = encoder.getScalars(temporalData)[0]

      assert isinstance(spatialData, (float, int))
      assert isinstance(temporalData, (float, int))
      output['spatialTopDownOut'][i] = spatialData
      output['temporalTopDownOut'][i] = temporalData

Response:
Convert all of the non - numeric fields from spatialOutput and temporalOutput
    into their scalar equivalents and records them in the output dictionary.


Instruction:
How would you explain what the following Python 3 function does
def getOutputElementCount(self, name):

    if name == "resetOut":
      print ("WARNING: getOutputElementCount should not have been called with "
             "resetOut")
      return 1

    elif name == "sequenceIdOut":
      print ("WARNING: getOutputElementCount should not have been called with "
             "sequenceIdOut")
      return 1

    elif name == "dataOut":
      if self.encoder is None:
        raise Exception("NuPIC requested output element count for 'dataOut' "
                        "on a RecordSensor node, but the encoder has not "
                        "been set")
      return self.encoder.getWidth()

    elif name == "sourceOut":
      if self.encoder is None:
        raise Exception("NuPIC requested output element count for 'sourceOut' "
                        "on a RecordSensor node, "
                        "but the encoder has not been set")
      return len(self.encoder.getDescription())

    elif name == "bucketIdxOut":
      return 1

    elif name == "actValueOut":
      return 1

    elif name == "categoryOut":
      return self.numCategories

    elif name == 'spatialTopDownOut' or name == 'temporalTopDownOut':
      if self.encoder is None:
        raise Exception("NuPIC requested output element count for 'sourceOut' "
                        "on a RecordSensor node, "
                        "but the encoder has not been set")
      return len(self.encoder.getDescription())
    else:
      raise Exception("Unknown output %s" % name)

Response: Returns the number of output elements in a single Sequences.


Instruction:
Can you implement a function in Python 3 that
sets the value of a Spec parameter.

Response:
def setParameter(self, parameterName, index, parameterValue):
    """
      Set the value of a Spec parameter. Most parameters are handled
      automatically by PyRegion's parameter set mechanism. The ones that need
      special treatment are explicitly handled here.
    """
    if parameterName == 'topDownMode':
      self.topDownMode = parameterValue
    elif parameterName == 'predictedField':
      self.predictedField = parameterValue
    else:
      raise Exception('Unknown parameter: ' + parameterName)


Instruction:
How would you implement a function in Python 3 that
writes this region to the given proto.

Response:
def writeToProto(self, proto):
    """
    Overrides :meth:`nupic.bindings.regions.PyRegion.PyRegion.writeToProto`.
    """
    self.encoder.write(proto.encoder)
    if self.disabledEncoder is not None:
      self.disabledEncoder.write(proto.disabledEncoder)
    proto.topDownMode = int(self.topDownMode)
    proto.verbosity = self.verbosity
    proto.numCategories = self.numCategories


Instruction:
Given the following Python 3 function, write the documentation
def readFromProto(cls, proto):
    instance = cls()

    instance.encoder = MultiEncoder.read(proto.encoder)
    if proto.disabledEncoder is not None:
      instance.disabledEncoder = MultiEncoder.read(proto.disabledEncoder)
    instance.topDownMode = bool(proto.topDownMode)
    instance.verbosity = proto.verbosity
    instance.numCategories = proto.numCategories

    return instance

Response:
Reads a
    from a proto.


Instruction:
Can you generate the documentation for the following Python 3 function
def computeAccuracy(model, size, top):
  accuracy = []

  # Load MSNBC web data file
  filename = os.path.join(os.path.dirname(__file__), "msnbc990928.zip")
  with zipfile.ZipFile(filename) as archive:
    with archive.open("msnbc990928.seq") as datafile:
      # Skip header lines (first 7 lines)
      for _ in xrange(7):
        next(datafile)

      # Skip learning data and compute accuracy using only new sessions
      for _ in xrange(LEARNING_RECORDS):
        next(datafile)

      # Compute prediction accuracy by checking if the next page in the sequence
      # is within the top N predictions calculated by the model
      for _ in xrange(size):
        pages = readUserSession(datafile)
        model.resetSequenceStates()
        for i in xrange(len(pages) - 1):
          result = model.run({"page": pages[i]})
          inferences = result.inferences["multiStepPredictions"][1]

          # Get top N predictions for the next page
          predicted = sorted(inferences.items(), key=itemgetter(1), reverse=True)[:top]

          # Check if the next page is within the predicted pages
          accuracy.append(1 if pages[i + 1] in zip(*predicted)[0] else 0)

  return np.mean(accuracy)

Response: Compute prediction accuracy using the given model and size.


Instruction:
Explain what the following Python 3 code does
def readUserSession(datafile):
  for line in datafile:
    pages = line.split()
    total = len(pages)
    # Select user sessions with 2 or more pages
    if total < 2:
      continue

    # Exclude outliers by removing extreme long sessions
    if total > 500:
      continue

    return [PAGE_CATEGORIES[int(i) - 1] for i in pages]
  return []

Response: Reads the user session record from the file s cursor position.


Instruction:
How would you explain what the following Python 3 function does
def rewind(self):

    # Superclass rewind
    super(FileRecordStream, self).rewind()

    self.close()
    self._file = open(self._filename, self._mode)
    self._reader = csv.reader(self._file, dialect="excel")

    # Skip header rows
    self._reader.next()
    self._reader.next()
    self._reader.next()

    # Reset record count, etc.
    self._recordCount = 0

Response: Rewinds the file to the beginning of the file.


Instruction:
Create a Python 3 function to
return the next available data record from the file.

Response:
def getNextRecord(self, useCache=True):
    """ Returns next available data record from the file.

    :returns: a data row (a list or tuple) if available; None, if no more
              records in the table (End of Stream - EOS); empty sequence (list
              or tuple) when timing out while waiting for the next record.
    """
    assert self._file is not None
    assert self._mode == self._FILE_READ_MODE

    # Read the line
    try:
      line = self._reader.next()

    except StopIteration:
      if self.rewindAtEOF:
        if self._recordCount == 0:
          raise Exception("The source configured to reset at EOF but "
                          "'%s' appears to be empty" % self._filename)
        self.rewind()
        line = self._reader.next()

      else:
        return None

    # Keep score of how many records were read
    self._recordCount += 1

    # Split the line to text fields and convert each text field to a Python
    # object if value is missing (empty string) encode appropriately for
    # upstream consumers in the case of numeric types, this means replacing
    # missing data with a sentinel value for string type, we can leave the empty
    # string in place
    record = []
    for i, f in enumerate(line):
      #print "DEBUG: Evaluating field @ index %s: %r" % (i, f)
      #sys.stdout.flush()
      if f in self._missingValues:
        record.append(SENTINEL_VALUE_FOR_MISSING_DATA)
      else:
        # either there is valid data, or the field is string type,
        # in which case the adapter does the right thing by default
        record.append(self._adapters[i](f))

    return record


Instruction:
Given the following Python 3 function, write the documentation
def appendRecord(self, record):

    assert self._file is not None
    assert self._mode == self._FILE_WRITE_MODE
    assert isinstance(record, (list, tuple)), \
      "unexpected record type: " + repr(type(record))

    assert len(record) == self._fieldCount, \
      "len(record): %s, fieldCount: %s" % (len(record), self._fieldCount)

    # Write header if needed
    if self._recordCount == 0:
      # Write the header
      names, types, specials = zip(*self.getFields())
      for line in names, types, specials:
        self._writer.writerow(line)

    # Keep track of sequences, make sure time flows forward
    self._updateSequenceInfo(record)

    line = [self._adapters[i](f) for i, f in enumerate(record)]

    self._writer.writerow(line)
    self._recordCount += 1

Response: Adds a record to the underlying CSV file.


Instruction:
Given the following Python 3 function, write the documentation
def appendRecords(self, records, progressCB=None):

    for record in records:
      self.appendRecord(record)
      if progressCB is not None:
        progressCB()

Response: Adds multiple records to the internal storage.


Instruction:
How would you explain what the following Python 3 function does
def getBookmark(self):

    if self._write and self._recordCount==0:
      return None

    rowDict = dict(filepath=os.path.realpath(self._filename),
                   currentRow=self._recordCount)
    return json.dumps(rowDict)

Response: Gets a bookmark or anchor to the current position.


Instruction:
How would you explain what the following Python 3 function does
def seekFromEnd(self, numRecords):
    self._file.seek(self._getTotalLineCount() - numRecords)
    return self.getBookmark()

Response: Seeks to the end of the file and returns a bookmark to the desired location.


Instruction:
Can you create a Python 3 function that
reads the file and returns a dictionary of stats.

Response:
def getStats(self):
    """
    Parse the file using dedicated reader and collect fields stats. Never
    called if user of :class:`~.FileRecordStream` does not invoke
    :meth:`~.FileRecordStream.getStats` method.

    :returns:
          a dictionary of stats. In the current implementation, min and max
          fields are supported. Example of the return dictionary is:

          .. code-block:: python

             {
               'min' : [f1_min, f2_min, None, None, fn_min],
               'max' : [f1_max, f2_max, None, None, fn_max]
             }

          (where fx_min/fx_max are set for scalar fields, or None if not)

    """

    # Collect stats only once per File object, use fresh csv iterator
    # to keep the next() method returning sequential records no matter when
    # caller asks for stats
    if self._stats == None:
      # Stats are only available when reading csv file
      assert self._mode == self._FILE_READ_MODE

      inFile = open(self._filename, self._FILE_READ_MODE)

      # Create a new reader; read names, types, specials
      reader = csv.reader(inFile, dialect="excel")
      names = [n.strip() for n in reader.next()]
      types = [t.strip() for t in reader.next()]
      # Skip over specials
      reader.next()

      # Initialize stats to all None
      self._stats = dict()
      self._stats['min'] = []
      self._stats['max'] = []

      for i in xrange(len(names)):
        self._stats['min'].append(None)
        self._stats['max'].append(None)

      # Read the file, collect stats
      while True:
        try:
          line = reader.next()
          for i, f in enumerate(line):
            if (len(types) > i and
                types[i] in [FieldMetaType.integer, FieldMetaType.float] and
                f not in self._missingValues):
              value = self._adapters[i](f)
              if self._stats['max'][i] == None or \
                 self._stats['max'][i] < value:
                self._stats['max'][i] = value
              if self._stats['min'][i] == None or \
                 self._stats['min'][i] > value:
                self._stats['min'][i] = value

        except StopIteration:
          break

    return self._stats


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _updateSequenceInfo(self, r):

    # Get current sequence id (if any)
    newSequence = False
    sequenceId = (r[self._sequenceIdIdx]
                  if self._sequenceIdIdx is not None else None)
    if sequenceId != self._currSequence:
      # verify that the new sequence didn't show up before
      if sequenceId in self._sequences:
        raise Exception('Broken sequence: %s, record: %s' % \
                        (sequenceId, r))

      # add the finished sequence to the set of sequence
      self._sequences.add(self._currSequence)
      self._currSequence = sequenceId

      # Verify that the reset is consistent (if there is one)
      if self._resetIdx:
        assert r[self._resetIdx] == 1
      newSequence = True

    else:
      # Check the reset
      reset = False
      if self._resetIdx:
        reset = r[self._resetIdx]
        if reset == 1:
          newSequence = True

    # If it's still the same old sequence make sure the time flows forward
    if not newSequence:
      if self._timeStampIdx and self._currTime is not None:
        t = r[self._timeStampIdx]
        if t < self._currTime:
          raise Exception('No time travel. Early timestamp for record: %s' % r)

    if self._timeStampIdx:
      self._currTime = r[self._timeStampIdx]

Response: Update the internal state of the sequence info in the current record.


Instruction:
Write a Python 3 script for
extracting start row from the bookmark information

Response:
def _getStartRow(self, bookmark):
    """ Extracts start row from the bookmark information
    """
    bookMarkDict = json.loads(bookmark)

    realpath = os.path.realpath(self._filename)

    bookMarkFile = bookMarkDict.get('filepath', None)

    if bookMarkFile != realpath:
      print ("Ignoring bookmark due to mismatch between File's "
             "filename realpath vs. bookmark; realpath: %r; bookmark: %r") % (
        realpath, bookMarkDict)
      return 0
    else:
      return bookMarkDict['currentRow']


Instruction:
Make a summary of the following Python 3 code
def _getTotalLineCount(self):
    # Flush the file before we open it again to count lines
    if self._mode == self._FILE_WRITE_MODE:
      self._file.flush()
    return sum(1 for line in open(self._filename, self._FILE_READ_MODE))

Response: Returns the number of lines in dataset including header lines


Instruction:
Given the following Python 3 function, write the documentation
def getDataRowCount(self):
    numLines = self._getTotalLineCount()

    if numLines == 0:
      # this may be the case in a file opened for write before the
      # header rows are written out
      assert self._mode == self._FILE_WRITE_MODE and self._recordCount == 0
      numDataRows = 0
    else:
      numDataRows = numLines - self._NUM_HEADER_ROWS

    assert numDataRows >= 0

    return numDataRows

Response: returns the number of data rows in the current dataset


Instruction:
Write a Python 3 function for
handling looping over the input data and passing each row into the given model object, as well as extracting the result object and passing it into an output handler. :param inputData: file path to input data CSV :param model: OPF Model object :param gymName: Gym name, used for output handler naming :param plot: Whether to use matplotlib or not. If false, uses file output.

Response:
def runIoThroughNupic(inputData, model, gymName, plot):
  """
  Handles looping over the input data and passing each row into the given model
  object, as well as extracting the result object and passing it into an output
  handler.
  :param inputData: file path to input data CSV
  :param model: OPF Model object
  :param gymName: Gym name, used for output handler naming
  :param plot: Whether to use matplotlib or not. If false, uses file output.
  """
  inputFile = open(inputData, "rb")
  csvReader = csv.reader(inputFile)
  # skip header rows
  csvReader.next()
  csvReader.next()
  csvReader.next()

  shifter = InferenceShifter()
  if plot:
    output = nupic_anomaly_output.NuPICPlotOutput(gymName)
  else:
    output = nupic_anomaly_output.NuPICFileOutput(gymName)

  counter = 0
  for row in csvReader:
    counter += 1
    if (counter % 100 == 0):
      print "Read %i lines..." % counter
    timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)
    consumption = float(row[1])
    result = model.run({
      "timestamp": timestamp,
      "kw_energy_consumption": consumption
    })

    if plot:
      result = shifter.shift(result)

    prediction = result.inferences["multiStepBestPredictions"][1]
    anomalyScore = result.inferences["anomalyScore"]
    output.write(timestamp, consumption, prediction, anomalyScore)

  inputFile.close()
  output.close()


Instruction:
Explain what the following Python 3 code does
def topDownCompute(self, encoded):

    #Decode to delta scalar
    if self._prevAbsolute==None or self._prevDelta==None:
      return [EncoderResult(value=0, scalar=0,
                             encoding=numpy.zeros(self.n))]
    ret = self._adaptiveScalarEnc.topDownCompute(encoded)
    if self._prevAbsolute != None:
      ret = [EncoderResult(value=ret[0].value+self._prevAbsolute,
                          scalar=ret[0].scalar+self._prevAbsolute,
                          encoding=ret[0].encoding)]
#      ret[0].value+=self._prevAbsolute
#      ret[0].scalar+=self._prevAbsolute
    return ret

Response: This method computes the top down of the scalar encoder.


Instruction:
How would you explain what the following Python 3 function does
def isTemporal(inferenceElement):
    if InferenceElement.__temporalInferenceElements is None:
      InferenceElement.__temporalInferenceElements = \
                                set([InferenceElement.prediction])

    return inferenceElement in InferenceElement.__temporalInferenceElements

Response:
Returns True if the inference element is predicted the input
    for the NEXT timestep.


Instruction:
Can you write a function in Python 3 where it
returns the number of records that elapse between when an inference is made and when the corresponding input record will appear.

Response:
def getTemporalDelay(inferenceElement, key=None):
    """ Returns the number of records that elapse between when an inference is
    made and when the corresponding input record will appear. For example, a
    multistep prediction for 3 timesteps out will have a delay of 3

    Parameters:
    -----------------------------------------------------------------------

    inferenceElement:   The InferenceElement value being delayed
    key:                If the inference is a dictionary type, this specifies
                        key for the sub-inference that is being delayed
    """
    # -----------------------------------------------------------------------
    # For next step prediction, we shift by 1
    if inferenceElement in (InferenceElement.prediction,
                            InferenceElement.encodings):
      return 1
    # -----------------------------------------------------------------------
    # For classification, anomaly scores, the inferences immediately succeed the
    # inputs
    if inferenceElement in (InferenceElement.anomalyScore,
                            InferenceElement.anomalyLabel,
                            InferenceElement.classification,
                            InferenceElement.classConfidences):
      return 0
    # -----------------------------------------------------------------------
    # For multistep prediction, the delay is based on the key in the inference
    # dictionary
    if inferenceElement in (InferenceElement.multiStepPredictions,
                            InferenceElement.multiStepBestPredictions):
      return int(key)

    # -----------------------------------------------------------------------
    # default: return 0
    return 0


Instruction:
Can you tell what is the following Python 3 function doing
def getMaxDelay(inferences):
    maxDelay = 0
    for inferenceElement, inference in inferences.iteritems():
      if isinstance(inference, dict):
        for key in inference.iterkeys():
          maxDelay = max(InferenceElement.getTemporalDelay(inferenceElement,
                                                            key),
                         maxDelay)
      else:
        maxDelay = max(InferenceElement.getTemporalDelay(inferenceElement),
                       maxDelay)

    return maxDelay

Response: Returns the maximum delay for the InferenceElements in the inferences dictionary


Instruction:
Given the following Python 3 function, write the documentation
def isTemporal(inferenceType):
    if InferenceType.__temporalInferenceTypes is None:
      InferenceType.__temporalInferenceTypes = \
                                set([InferenceType.TemporalNextStep,
                                     InferenceType.TemporalClassification,
                                     InferenceType.TemporalAnomaly,
                                     InferenceType.TemporalMultiStep,
                                     InferenceType.NontemporalMultiStep])

    return inferenceType in InferenceType.__temporalInferenceTypes

Response: Returns True if the inference type is temporal.


Instruction:
Write a Python 3 script to
make a directory from the given absolute path.

Response:
def makeDirectoryFromAbsolutePath(absDirPath):
  """ Makes directory for the given directory path with default permissions.
  If the directory already exists, it is treated as success.

  absDirPath:   absolute path of the directory to create.

  Returns:      absDirPath arg

  Exceptions:         OSError if directory creation fails
  """

  assert os.path.isabs(absDirPath)

  try:
    os.makedirs(absDirPath)
  except OSError, e:
    if e.errno != os.errno.EEXIST:
      raise

  return absDirPath


Instruction:
Implement a function in Python 3 to
read in the given configuration file and return a dict describing the file.

Response:
def _readConfigFile(cls, filename, path=None):
    """ Parse the given XML file and return a dict describing the file.

    Parameters:
    ----------------------------------------------------------------
    filename:  name of XML file to parse (no path)
    path:      path of the XML file. If None, then use the standard
                  configuration search path.
    retval:    returns a dict with each property as a key and a dict of all
               the property's attributes as value
    """

    outputProperties = dict()

    # Get the path to the config files.
    if path is None:
      filePath = cls.findConfigFile(filename)
    else:
      filePath = os.path.join(path, filename)

    # ------------------------------------------------------------------
    # Read in the config file
    try:
      if filePath is not None:
        try:
          # Use warn since console log level is set to warning
          _getLoggerBase().debug("Loading config file: %s", filePath)
          with open(filePath, 'r') as inp:
            contents = inp.read()
        except Exception:
          raise RuntimeError("Expected configuration file at %s" % filePath)
      else:
        # If the file was not found in the normal search paths, which includes
        # checking the NTA_CONF_PATH, we'll try loading it from pkg_resources.
        try:
          contents = resource_string("nupic.support", filename)
        except Exception as resourceException:
          # We expect these to be read, and if they don't exist we'll just use
          # an empty configuration string.
          if filename in [USER_CONFIG, CUSTOM_CONFIG]:
            contents = '<configuration/>'
          else:
            raise resourceException

      elements = ElementTree.XML(contents)

      if elements.tag != 'configuration':
        raise RuntimeError("Expected top-level element to be 'configuration' "
                           "but got '%s'" % (elements.tag))

      # ------------------------------------------------------------------
      # Add in each property found
      propertyElements = elements.findall('./property')

      for propertyItem in propertyElements:

        propInfo = dict()

        # Parse this property element
        propertyAttributes = list(propertyItem)
        for propertyAttribute in propertyAttributes:
          propInfo[propertyAttribute.tag] = propertyAttribute.text

        # Get the name
        name = propInfo.get('name', None)

        # value is allowed to be empty string
        if 'value' in propInfo and propInfo['value'] is None:
          value = ''
        else:
          value = propInfo.get('value', None)

          if value is None:
            if 'novalue' in propInfo:
              # Placeholder "novalue" properties are intended to be overridden
              # via dynamic configuration or another configuration layer.
              continue
            else:
              raise RuntimeError("Missing 'value' element within the property "
                                 "element: => %s " % (str(propInfo)))

        # The value is allowed to contain substitution tags of the form
        # ${env.VARNAME}, which should be substituted with the corresponding
        # environment variable values
        restOfValue = value
        value = ''
        while True:
          # Find the beginning of substitution tag
          pos = restOfValue.find('${env.')
          if pos == -1:
            # No more environment variable substitutions
            value += restOfValue
            break

          # Append prefix to value accumulator
          value += restOfValue[0:pos]

          # Find the end of current substitution tag
          varTailPos = restOfValue.find('}', pos)
          if varTailPos == -1:
            raise RuntimeError(
              "Trailing environment variable tag delimiter '}'"
              " not found in %r" % (restOfValue))

          # Extract environment variable name from tag
          varname = restOfValue[pos + 6:varTailPos]
          if varname not in os.environ:
            raise RuntimeError("Attempting to use the value of the environment"
                               " variable %r, which is not defined" % (
                               varname))
          envVarValue = os.environ[varname]

          value += envVarValue

          restOfValue = restOfValue[varTailPos + 1:]

        # Check for errors
        if name is None:
          raise RuntimeError(
            "Missing 'name' element within following property "
            "element:\n => %s " % (str(propInfo)))

        propInfo['value'] = value
        outputProperties[name] = propInfo

      return outputProperties
    except Exception:
      _getLoggerBase().exception("Error while parsing configuration file: %s.",
                             filePath)
      raise


Instruction:
Create a Python 3 function for
setting multiple custom properties and persist them to the custom configuration store.

Response:
def setCustomProperties(cls, properties):
    """ Set multiple custom properties and persist them to the custom
    configuration store.

    Parameters:
    ----------------------------------------------------------------
    properties: a dict of property name/value pairs to set
    """
    _getLogger().info("Setting custom configuration properties=%r; caller=%r",
                      properties, traceback.format_stack())

    _CustomConfigurationFileWrapper.edit(properties)

    for propertyName, value in properties.iteritems():
      cls.set(propertyName, value)


Instruction:
Create a Python 3 function to
clear all properties from in - memory cache but do NOT alter the custom configuration file.

Response:
def clear(cls):
    """ Clear all configuration properties from in-memory cache, but do NOT
    alter the custom configuration file. Used in unit-testing.
    """
    # Clear the in-memory settings cache, forcing reload upon subsequent "get"
    # request.
    super(Configuration, cls).clear()

    # Reset in-memory custom configuration info.
    _CustomConfigurationFileWrapper.clear(persistent=False)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def resetCustomConfig(cls):
    _getLogger().info("Resetting all custom configuration properties; "
                      "caller=%r", traceback.format_stack())

    # Clear the in-memory settings cache, forcing reload upon subsequent "get"
    # request.
    super(Configuration, cls).clear()

    # Delete the persistent custom configuration store and reset in-memory
    # custom configuration info
    _CustomConfigurationFileWrapper.clear(persistent=True)

Response: Clear all custom configuration settings and delete the persistent custom configuration store.


Instruction:
Given the following Python 3 function, write the documentation
def clear(cls, persistent=False):
    if persistent:
      try:
        os.unlink(cls.getPath())
      except OSError, e:
        if e.errno != errno.ENOENT:
          _getLogger().exception("Error %s while trying to remove dynamic " \
                                 "configuration file: %s", e.errno,
                                 cls.getPath())
          raise
    cls._path = None

Response: Delete the temporary file containing the dynamic configuration file.


Instruction:
How would you explain what the following Python 3 function does
def getCustomDict(cls):
    if not os.path.exists(cls.getPath()):
      return dict()

    properties = Configuration._readConfigFile(os.path.basename(
      cls.getPath()), os.path.dirname(cls.getPath()))

    values = dict()
    for propName in properties:
      if 'value' in properties[propName]:
        values[propName] = properties[propName]['value']

    return values

Response: Returns a dict of all temporary values in custom configuration file


Instruction:
Create a Python 3 function to
edit the XML configuration file with the parameters specified by properties.

Response:
def edit(cls, properties):
    """ Edits the XML configuration file with the parameters specified by
    properties

    Parameters:
    ----------------------------------------------------------------
    properties: dict of settings to be applied to the custom configuration store
                 (key is property name, value is value)
    """
    copyOfProperties = copy(properties)

    configFilePath = cls.getPath()

    try:
      with open(configFilePath, 'r') as fp:
        contents = fp.read()
    except IOError, e:
      if e.errno != errno.ENOENT:
        _getLogger().exception("Error %s reading custom configuration store "
                               "from %s, while editing properties %s.",
                               e.errno, configFilePath, properties)
        raise
      contents = '<configuration/>'

    try:
      elements = ElementTree.XML(contents)
      ElementTree.tostring(elements)
    except Exception, e:
      # Raising error as RuntimeError with custom message since ElementTree
      # exceptions aren't clear.
      msg = "File contents of custom configuration is corrupt.  File " \
            "location: %s; Contents: '%s'. Original Error (%s): %s." % \
            (configFilePath, contents, type(e), e)
      _getLogger().exception(msg)
      raise RuntimeError(msg), None, sys.exc_info()[2]

    if elements.tag != 'configuration':
      e = "Expected top-level element to be 'configuration' but got '%s'" % \
          (elements.tag)
      _getLogger().error(e)
      raise RuntimeError(e)

    # Apply new properties to matching settings in the custom config store;
    # pop matching properties from our copy of the properties dict
    for propertyItem in elements.findall('./property'):
      propInfo = dict((attr.tag, attr.text) for attr in propertyItem)
      name = propInfo['name']
      if name in copyOfProperties:
        foundValues = propertyItem.findall('./value')
        if len(foundValues) > 0:
          foundValues[0].text = str(copyOfProperties.pop(name))
          if not copyOfProperties:
            break
        else:
          e = "Property %s missing value tag." % (name,)
          _getLogger().error(e)
          raise RuntimeError(e)

    # Add unmatched remaining properties to custom config store
    for propertyName, value in copyOfProperties.iteritems():
      newProp = ElementTree.Element('property')
      nameTag = ElementTree.Element('name')
      nameTag.text = propertyName
      newProp.append(nameTag)

      valueTag = ElementTree.Element('value')
      valueTag.text = str(value)
      newProp.append(valueTag)

      elements.append(newProp)

    try:
      makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))
      with open(configFilePath, 'w') as fp:
        fp.write(ElementTree.tostring(elements))
    except Exception, e:
      _getLogger().exception("Error while saving custom configuration "
                             "properties %s in %s.", properties,
                             configFilePath)
      raise


Instruction:
Make a summary of the following Python 3 code
def _setPath(cls):
    cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'],
                             cls.customFileName)

Response: Sets the path of the custom configuration file in the current language.


Instruction:
Write a Python 3 function that can
get the particle state as a dict. This is enough information to acquire this particle on another worker.

Response:
def getState(self):
    """Get the particle state as a dict. This is enough information to
    instantiate this particle on another worker."""
    varStates = dict()
    for varName, var in self.permuteVars.iteritems():
      varStates[varName] = var.getState()

    return dict(id=self.particleId,
                genIdx=self.genIdx,
                swarmId=self.swarmId,
                varStates=varStates)


Instruction:
Can you generate the documentation for the following Python 3 function
def initStateFrom(self, particleId, particleState, newBest):
    # Get the update best position and result?
    if newBest:
      (bestResult, bestPosition) = self._resultsDB.getParticleBest(particleId)
    else:
      bestResult = bestPosition = None

    # Replace with the position and velocity of each variable from
    #  saved state
    varStates = particleState['varStates']
    for varName in varStates.keys():
      varState = copy.deepcopy(varStates[varName])
      if newBest:
        varState['bestResult'] = bestResult
      if bestPosition is not None:
        varState['bestPosition'] = bestPosition[varName]
      self.permuteVars[varName].setState(varState)

Response: Initialize all of our variables positions velocities and bestResult and bestPosition from the given particle state.


Instruction:
Write a Python 3 function that can
copy specific variables from particleState into this particle.

Response:
def copyVarStatesFrom(self, particleState, varNames):
    """Copy specific variables from particleState into this particle.

    Parameters:
    --------------------------------------------------------------
    particleState:        dict produced by a particle's getState() method
    varNames:             which variables to copy
    """
    # Set this to false if you don't want the variable to move anymore
    #  after we set the state
    allowedToMove = True

    for varName in particleState['varStates']:
      if varName in varNames:

        # If this particle doesn't include this field, don't copy it
        if varName not in self.permuteVars:
          continue

        # Set the best position to the copied position
        state = copy.deepcopy(particleState['varStates'][varName])
        state['_position'] = state['position']
        state['bestPosition'] = state['position']

        if not allowedToMove:
          state['velocity'] = 0

        # Set the state now
        self.permuteVars[varName].setState(state)

        if allowedToMove:
          # Let the particle move in both directions from the best position
          #  it found previously and set it's initial velocity to a known
          #  fraction of the total distance.
          self.permuteVars[varName].resetVelocity(self._rng)


Instruction:
Implement a Python 3 function for
returning the position of this particle. This returns a dictionary of key - value pairs where each key is the name of the flattened permutation variable and each value is the value of the flattened permutation variable.

Response:
def getPosition(self):
    """Return the position of this particle. This returns a dict() of key
    value pairs where each key is the name of the flattened permutation
    variable and the value is its chosen value.

    Parameters:
    --------------------------------------------------------------
    retval:     dict() of flattened permutation choices
    """
    result = dict()
    for (varName, value) in self.permuteVars.iteritems():
      result[varName] = value.getPosition()

    return result


Instruction:
Can you create a Python 3 function that
returns the position of a particle given its state dict.

Response:
def getPositionFromState(pState):
    """Return the position of a particle given its state dict.

    Parameters:
    --------------------------------------------------------------
    retval:     dict() of particle position, keys are the variable names,
                  values are their positions
    """
    result = dict()
    for (varName, value) in pState['varStates'].iteritems():
      result[varName] = value['position']

    return result


Instruction:
Can you write a function in Python 3 where it
agitates this particle so that it is likely to go to a new position.

Response:
def agitate(self):
    """Agitate this particle so that it is likely to go to a new position.
    Every time agitate is called, the particle is jiggled an even greater
    amount.

    Parameters:
    --------------------------------------------------------------
    retval:               None
    """
    for (varName, var) in self.permuteVars.iteritems():
      var.agitate()

    self.newPosition()


Instruction:
How would you code a function in Python 3 to
choose a new position based on the results obtained so far from all other variables.

Response:
def newPosition(self, whichVars=None):
    # TODO: incorporate data from choice variables....
    # TODO: make sure we're calling this when appropriate.
    """Choose a new position based on results obtained so far from all other
    particles.

    Parameters:
    --------------------------------------------------------------
    whichVars:       If not None, only move these variables
    retval:               new position
    """
    # Get the global best position for this swarm generation
    globalBestPosition = None
    # If speculative particles are enabled, use the global best considering
    #  even particles in the current generation. This gives better results
    #  but does not provide repeatable results because it depends on
    #  worker timing
    if self._hsObj._speculativeParticles:
      genIdx = self.genIdx
    else:
      genIdx = self.genIdx - 1

    if genIdx >= 0:
      (bestModelId, _) = self._resultsDB.bestModelIdAndErrScore(self.swarmId,
                                                                genIdx)
      if bestModelId is not None:
        (particleState, _, _, _, _) = self._resultsDB.getParticleInfo(
          bestModelId)
        globalBestPosition = Particle.getPositionFromState(particleState)

    # Update each variable
    for (varName, var) in self.permuteVars.iteritems():
      if whichVars is not None and varName not in whichVars:
        continue
      if globalBestPosition is None:
        var.newPosition(None, self._rng)
      else:
        var.newPosition(globalBestPosition[varName], self._rng)

    # get the new position
    position = self.getPosition()

    # Log the new position
    if self.logger.getEffectiveLevel() <= logging.DEBUG:
      msg = StringIO.StringIO()
      print >> msg, "New particle position: \n%s" % (pprint.pformat(position,
                                                                    indent=4))
      print >> msg, "Particle variables:"
      for (varName, var) in self.permuteVars.iteritems():
        print >> msg, "  %s: %s" % (varName, str(var))
      self.logger.debug(msg.getvalue())
      msg.close()

    return position


Instruction:
Here you have a function in Python 3, explain what it does
def propose(self, current, r):
    stay = (r.uniform(0, 1) < self.kernel)
    if stay:
      logKernel = numpy.log(self.kernel)
      return current, logKernel, logKernel
    else: # Choose uniformly, not according to the pmf.
      curIndex = self.keyMap[current]
      ri = r.randint(0, self.nKeys-1)
      logKernel = numpy.log(1.0 - self.kernel)
      lp = logKernel + self.logp
      if ri < curIndex: return self.keys[ri], lp, lp
      else: return self.keys[ri+1], lp, lp

Response:
Generates a random sample from the discrete probability distribution and
    returns its value the log of the probability of sampling that value and the log of the probability of sampling that value and the log of the probability of sampling that value.


Instruction:
Write a Python 3 script to
generate a random sample from the Poisson probability distribution withlocation and scale equal to the current value.

Response:
def propose(self, current, r):
    """Generates a random sample from the Poisson probability distribution with
    with location and scale parameter equal to the current value (passed in).
    Returns the value of the random sample, the log of the probability of
    sampling that value, and the log of the probability of sampling the current
    value if the roles of the new sample and the current sample were reversed
    (the log of the backward proposal probability).
    """
    curLambda = current + self.offset
    x, logProb = PoissonDistribution(curLambda).sample(r)
    logBackward = PoissonDistribution(x+self.offset).logDensity(current)
    return x, logProb, logBackward


Instruction:
Can you generate the documentation for the following Python 3 function
def __getLogger(cls):
    if cls.__logger is None:
      cls.__logger = opf_utils.initLogger(cls)
    return cls.__logger

Response: Returns the logger for this object.


Instruction:
Implement a Python 3 function for
creating a new model instance given a dictionary describing the current model and the log level.

Response:
def create(modelConfig, logLevel=logging.ERROR):
    """ Create a new model instance, given a description dictionary.

    :param modelConfig: (dict)
           A dictionary describing the current model,
           `described here <../../quick-start/example-model-params.html>`_.

    :param logLevel: (int) The level of logging output that should be generated

    :raises Exception: Unsupported model type

    :returns: :class:`nupic.frameworks.opf.model.Model`
    """
    logger = ModelFactory.__getLogger()
    logger.setLevel(logLevel)
    logger.debug("ModelFactory returning Model from dict: %s", modelConfig)

    modelClass = None
    if modelConfig['model'] == "HTMPrediction":
      modelClass = HTMPredictionModel
    elif modelConfig['model'] == "TwoGram":
      modelClass = TwoGramModel
    elif modelConfig['model'] == "PreviousValue":
      modelClass = PreviousValueModel
    else:
      raise Exception("ModelFactory received unsupported Model type: %s" % \
                      modelConfig['model'])

    return modelClass(**modelConfig['modelParams'])


Instruction:
Write a Python 3 script for
loading the saved model from the directory specified by the checkpoint.

Response:
def loadFromCheckpoint(savedModelDir, newSerialization=False):
    """ Load saved model.

    :param savedModelDir: (string)
           Directory of where the experiment is to be or was saved
    :returns: (:class:`nupic.frameworks.opf.model.Model`) The loaded model
              instance.
    """
    if newSerialization:
      return HTMPredictionModel.readFromCheckpoint(savedModelDir)
    else:
      return Model.load(savedModelDir)


Instruction:
Here you have a function in Python 3, explain what it does
def compute(self, activeColumns, learn=True):
    self.activateCells(sorted(activeColumns), learn)
    self.activateDendrites(learn)

Response: Performs one time step of the Temporal Memory algorithm.


Instruction:
Write a Python 3 script for
activating the active cells using the current active columns and dendrite segments.

Response:
def activateCells(self, activeColumns, learn=True):
    """
    Calculate the active cells, using the current active columns and dendrite
    segments. Grow and reinforce synapses.

    :param activeColumns: (iter) A sorted list of active column indices.

    :param learn: (bool) If true, reinforce / punish / grow synapses.

      **Pseudocode:**

      ::

        for each column
          if column is active and has active distal dendrite segments
            call activatePredictedColumn
          if column is active and doesn't have active distal dendrite segments
            call burstColumn
          if column is inactive and has matching distal dendrite segments
            call punishPredictedColumn
    """
    prevActiveCells = self.activeCells
    prevWinnerCells = self.winnerCells
    self.activeCells = []
    self.winnerCells = []

    segToCol = lambda segment: int(segment.cell / self.cellsPerColumn)
    identity = lambda x: x

    for columnData in groupby2(activeColumns, identity,
                               self.activeSegments, segToCol,
                               self.matchingSegments, segToCol):
      (column,
       activeColumns,
       columnActiveSegments,
       columnMatchingSegments) = columnData
      if activeColumns is not None:
        if columnActiveSegments is not None:
          cellsToAdd = self.activatePredictedColumn(column,
                                                    columnActiveSegments,
                                                    columnMatchingSegments,
                                                    prevActiveCells,
                                                    prevWinnerCells,
                                                    learn)

          self.activeCells += cellsToAdd
          self.winnerCells += cellsToAdd
        else:
          (cellsToAdd,
           winnerCell) = self.burstColumn(column,
                                          columnMatchingSegments,
                                          prevActiveCells,
                                          prevWinnerCells,
                                          learn)

          self.activeCells += cellsToAdd
          self.winnerCells.append(winnerCell)
      else:
        if learn:
          self.punishPredictedColumn(column,
                                     columnActiveSegments,
                                     columnMatchingSegments,
                                     prevActiveCells,
                                     prevWinnerCells)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def activateDendrites(self, learn=True):
    (numActiveConnected,
     numActivePotential) = self.connections.computeActivity(
       self.activeCells,
       self.connectedPermanence)

    activeSegments = (
      self.connections.segmentForFlatIdx(i)
      for i in xrange(len(numActiveConnected))
      if numActiveConnected[i] >= self.activationThreshold
    )

    matchingSegments = (
      self.connections.segmentForFlatIdx(i)
      for i in xrange(len(numActivePotential))
      if numActivePotential[i] >= self.minThreshold
    )

    self.activeSegments = sorted(activeSegments,
                                 key=self.connections.segmentPositionSortKey)
    self.matchingSegments = sorted(matchingSegments,
                                   key=self.connections.segmentPositionSortKey)
    self.numActiveConnectedSynapsesForSegment = numActiveConnected
    self.numActivePotentialSynapsesForSegment = numActivePotential

    if learn:
      for segment in self.activeSegments:
        self.lastUsedIterationForSegment[segment.flatIdx] = self.iteration
      self.iteration += 1

Response: Activates the dendrites in the current active cells.


Instruction:
How would you explain what the following Python 3 function does
def reset(self):
    self.activeCells = []
    self.winnerCells = []
    self.activeSegments = []
    self.matchingSegments = []

Response: Resets the internal state of the sequence to its initial state.


Instruction:
Given the following Python 3 function, write the documentation
def activatePredictedColumn(self, column, columnActiveSegments,
                              columnMatchingSegments, prevActiveCells,
                              prevWinnerCells, learn):
    """
    Determines which cells in a predicted column should be added to winner cells
    list, and learns on the segments that correctly predicted this column.

    :param column: (int) Index of bursting column.

    :param columnActiveSegments: (iter) Active segments in this column.

    :param columnMatchingSegments: (iter) Matching segments in this column.

    :param prevActiveCells: (list) Active cells in ``t-1``.

    :param prevWinnerCells: (list) Winner cells in ``t-1``.

    :param learn: (bool) If true, grow and reinforce synapses.

    :returns: (list) A list of predicted cells that will be added to
              active cells and winner cells.
    """
    return self._activatePredictedColumn(
      self.connections, self._random,
      columnActiveSegments, prevActiveCells, prevWinnerCells,
      self.numActivePotentialSynapsesForSegment,
      self.maxNewSynapseCount, self.initialPermanence,
      self.permanenceIncrement, self.permanenceDecrement,
      self.maxSynapsesPerSegment, learn)

Response: Activates a column and returns a list of active cells and winner cells.


Instruction:
How would you implement a function in Python 3 that
punishes the Segments that incorrectly predicted a column to be active.

Response:
def punishPredictedColumn(self, column, columnActiveSegments,
                            columnMatchingSegments, prevActiveCells,
                            prevWinnerCells):
    """
    Punishes the Segments that incorrectly predicted a column to be active.

    :param column: (int) Index of bursting column.

    :param columnActiveSegments: (iter) Active segments for this column, or None
           if there aren't any.

    :param columnMatchingSegments: (iter) Matching segments for this column, or
           None if there aren't any.

    :param prevActiveCells: (list) Active cells in ``t-1``.

    :param prevWinnerCells: (list) Winner cells in ``t-1``.

    """
    self._punishPredictedColumn(
      self.connections, columnMatchingSegments, prevActiveCells,
      self.predictedSegmentDecrement)


Instruction:
Here you have a function in Python 3, explain what it does
def createSegment(self, cell):
    return self._createSegment(
      self.connections, self.lastUsedIterationForSegment, cell, self.iteration,
      self.maxSegmentsPerCell)

Response: Creates a segment on the specified cell.


Instruction:
Can you generate the documentation for the following Python 3 function
def _activatePredictedColumn(cls, connections, random, columnActiveSegments,
                               prevActiveCells, prevWinnerCells,
                               numActivePotentialSynapsesForSegment,
                               maxNewSynapseCount, initialPermanence,
                               permanenceIncrement, permanenceDecrement,
                               maxSynapsesPerSegment, learn):
    """
    :param connections: (Object)
    Connections for the TM. Gets mutated.

    :param random: (Object)
    Random number generator. Gets mutated.

    :param columnActiveSegments: (iter)
    Active segments in this column.

    :param prevActiveCells: (list)
    Active cells in `t-1`.

    :param prevWinnerCells: (list)
    Winner cells in `t-1`.

    :param numActivePotentialSynapsesForSegment: (list)
    Number of active potential synapses per segment, indexed by the segment's
    flatIdx.

    :param maxNewSynapseCount: (int)
    The maximum number of synapses added to a segment during learning

    :param initialPermanence: (float)
    Initial permanence of a new synapse.

    @permanenceIncrement (float)
    Amount by which permanences of synapses are incremented during learning.

    @permanenceDecrement (float)
    Amount by which permanences of synapses are decremented during learning.

    :param maxSynapsesPerSegment: (int)
    The maximum number of synapses per segment.

    :param learn: (bool)
    If true, grow and reinforce synapses.

    :returns: cellsToAdd (list)
    A list of predicted cells that will be added to active cells and winner
    cells.

    Pseudocode:
    for each cell in the column that has an active distal dendrite segment
      mark the cell as active
      mark the cell as a winner cell
      (learning) for each active distal dendrite segment
        strengthen active synapses
        weaken inactive synapses
        grow synapses to previous winner cells
    """

    cellsToAdd = []
    previousCell = None
    for segment in columnActiveSegments:
      if segment.cell != previousCell:
        cellsToAdd.append(segment.cell)
        previousCell = segment.cell

      if learn:
        cls._adaptSegment(connections, segment, prevActiveCells,
                          permanenceIncrement, permanenceDecrement)

        active = numActivePotentialSynapsesForSegment[segment.flatIdx]
        nGrowDesired = maxNewSynapseCount - active

        if nGrowDesired > 0:
          cls._growSynapses(connections, random, segment, nGrowDesired,
                            prevWinnerCells, initialPermanence,
                            maxSynapsesPerSegment)

    return cellsToAdd

Response: Activates the predicted column.


Instruction:
Here you have a function in Python 3, explain what it does
def _burstColumn(cls, connections, random, lastUsedIterationForSegment,
                   column, columnMatchingSegments, prevActiveCells,
                   prevWinnerCells, cellsForColumn,
                   numActivePotentialSynapsesForSegment, iteration,
                   maxNewSynapseCount, initialPermanence, permanenceIncrement,
                   permanenceDecrement, maxSegmentsPerCell,
                   maxSynapsesPerSegment, learn):
    """
    :param connections: (Object)
    Connections for the TM. Gets mutated.

    :param random: (Object)
    Random number generator. Gets mutated.

    :param lastUsedIterationForSegment: (list)
    Last used iteration for each segment, indexed by the segment's flatIdx.
    Gets mutated.

    :param column: (int)
    Index of bursting column.

    :param columnMatchingSegments: (iter)
    Matching segments in this column.

    :param prevActiveCells: (list)
    Active cells in `t-1`.

    :param prevWinnerCells: (list)
    Winner cells in `t-1`.

    :param cellsForColumn: (sequence)
    Range of cell indices on which to operate.

    :param numActivePotentialSynapsesForSegment: (list)
    Number of active potential synapses per segment, indexed by the segment's
    flatIdx.

    :param iteration: (int)
    The current timestep.

    :param maxNewSynapseCount: (int)
    The maximum number of synapses added to a segment during learning.

    :param initialPermanence: (float)
    Initial permanence of a new synapse.

    :param permanenceIncrement: (float)
    Amount by which permanences of synapses are incremented during learning.

    :param permanenceDecrement: (float)
    Amount by which permanences of synapses are decremented during learning.

    :param maxSegmentsPerCell: (int)
    The maximum number of segments per cell.

    :param maxSynapsesPerSegment: (int)
    The maximum number of synapses per segment.

    :param learn: (bool)
    Whether or not learning is enabled.

    :returns: (tuple) Contains:
                      `cells`         (iter),
                      `winnerCell`    (int),

    Pseudocode:
    mark all cells as active
    if there are any matching distal dendrite segments
      find the most active matching segment
      mark its cell as a winner cell
      (learning)
        grow and reinforce synapses to previous winner cells
    else
      find the cell with the least segments, mark it as a winner cell
      (learning)
        (optimization) if there are prev winner cells
          add a segment to this winner cell
          grow synapses to previous winner cells
    """
    if columnMatchingSegments is not None:
      numActive = lambda s: numActivePotentialSynapsesForSegment[s.flatIdx]
      bestMatchingSegment = max(columnMatchingSegments, key=numActive)
      winnerCell = bestMatchingSegment.cell

      if learn:
        cls._adaptSegment(connections, bestMatchingSegment, prevActiveCells,
                          permanenceIncrement, permanenceDecrement)

        nGrowDesired = maxNewSynapseCount - numActive(bestMatchingSegment)

        if nGrowDesired > 0:
          cls._growSynapses(connections, random, bestMatchingSegment,
                            nGrowDesired, prevWinnerCells, initialPermanence,
                            maxSynapsesPerSegment)
    else:
      winnerCell = cls._leastUsedCell(random, cellsForColumn, connections)
      if learn:
        nGrowExact = min(maxNewSynapseCount, len(prevWinnerCells))
        if nGrowExact > 0:
          segment = cls._createSegment(connections,
                                       lastUsedIterationForSegment, winnerCell,
                                       iteration, maxSegmentsPerCell)
          cls._growSynapses(connections, random, segment, nGrowExact,
                            prevWinnerCells, initialPermanence,
                            maxSynapsesPerSegment)

    return cellsForColumn, winnerCell

Response: This method is used to create a bursting column.


Instruction:
Implement a Python 3 function for
punishing the predicted column.

Response:
def _punishPredictedColumn(cls, connections, columnMatchingSegments,
                             prevActiveCells, predictedSegmentDecrement):
    """
    :param connections: (Object)
    Connections for the TM. Gets mutated.

    :param columnMatchingSegments: (iter)
    Matching segments for this column.

    :param prevActiveCells: (list)
    Active cells in `t-1`.

    :param predictedSegmentDecrement: (float)
    Amount by which segments are punished for incorrect predictions.

    Pseudocode:
    for each matching segment in the column
      weaken active synapses
    """
    if predictedSegmentDecrement > 0.0 and columnMatchingSegments is not None:
      for segment in columnMatchingSegments:
        cls._adaptSegment(connections, segment, prevActiveCells,
                          -predictedSegmentDecrement, 0.0)


Instruction:
Can you tell what is the following Python 3 function doing
def _createSegment(cls, connections, lastUsedIterationForSegment, cell,
                     iteration, maxSegmentsPerCell):
    """
    Create a segment on the connections, enforcing the maxSegmentsPerCell
    parameter.
    """
    # Enforce maxSegmentsPerCell.
    while connections.numSegments(cell) >= maxSegmentsPerCell:
      leastRecentlyUsedSegment = min(
        connections.segmentsForCell(cell),
        key=lambda segment : lastUsedIterationForSegment[segment.flatIdx])

      connections.destroySegment(leastRecentlyUsedSegment)

    # Create the segment.
    segment = connections.createSegment(cell)

    # Do TM-specific bookkeeping for the segment.
    if segment.flatIdx == len(lastUsedIterationForSegment):
      lastUsedIterationForSegment.append(iteration)
    elif segment.flatIdx < len(lastUsedIterationForSegment):
      # A flatIdx was recycled.
      lastUsedIterationForSegment[segment.flatIdx] = iteration
    else:
      raise AssertionError(
        "All segments should be created with the TM createSegment method.")

    return segment

Response: Create a segment on the connections and enforcing the maxSegmentsPerCell parameter.


Instruction:
Make a summary of the following Python 3 code
def _destroyMinPermanenceSynapses(cls, connections, random, segment,
                                    nDestroy, excludeCells):
    """
    Destroy nDestroy synapses on the specified segment, but don't destroy
    synapses to the "excludeCells".
    """

    destroyCandidates = sorted(
      (synapse for synapse in connections.synapsesForSegment(segment)
       if synapse.presynapticCell not in excludeCells),
      key=lambda s: s._ordinal
    )

    for _ in xrange(nDestroy):
      if len(destroyCandidates) == 0:
        break

      minSynapse = None
      minPermanence = float("inf")

      for synapse in destroyCandidates:
        if synapse.permanence < minPermanence - EPSILON:
          minSynapse = synapse
          minPermanence = synapse.permanence

      connections.destroySynapse(minSynapse)
      destroyCandidates.remove(minSynapse)

Response: Destroy nDestroy synapses on the specified segment but don t destroy synapses to the excludeCells.


Instruction:
Can you create a Python 3 function that
returns the cell with the smallest number of segments.

Response:
def _leastUsedCell(cls, random, cells, connections):
    """
    Gets the cell with the smallest number of segments.
    Break ties randomly.

    :param random: (Object)
    Random number generator. Gets mutated.

    :param cells: (list)
    Indices of cells.

    :param connections: (Object)
    Connections instance for the TM.

    :returns: (int) Cell index.
    """
    leastUsedCells = []
    minNumSegments = float("inf")
    for cell in cells:
      numSegments = connections.numSegments(cell)

      if numSegments < minNumSegments:
        minNumSegments = numSegments
        leastUsedCells = []

      if numSegments == minNumSegments:
        leastUsedCells.append(cell)

    i = random.getUInt32(len(leastUsedCells))
    return leastUsedCells[i]


Instruction:
How would you implement a function in Python 3 that
grows synapses on the segment.

Response:
def _growSynapses(cls, connections, random, segment, nDesiredNewSynapes,
                    prevWinnerCells, initialPermanence, maxSynapsesPerSegment):
    """
    Creates nDesiredNewSynapes synapses on the segment passed in if
    possible, choosing random cells from the previous winner cells that are
    not already on the segment.

    :param connections:        (Object) Connections instance for the tm
    :param random:             (Object) TM object used to generate random
                                        numbers
    :param segment:            (int)    Segment to grow synapses on.
    :param nDesiredNewSynapes: (int)    Desired number of synapses to grow
    :param prevWinnerCells:    (list)   Winner cells in `t-1`
    :param initialPermanence:  (float)  Initial permanence of a new synapse.

    """
    candidates = list(prevWinnerCells)

    for synapse in connections.synapsesForSegment(segment):
      i = binSearch(candidates, synapse.presynapticCell)
      if i != -1:
        del candidates[i]

    nActual = min(nDesiredNewSynapes, len(candidates))

    # Check if we're going to surpass the maximum number of synapses.
    overrun = connections.numSynapses(segment) + nActual - maxSynapsesPerSegment
    if overrun > 0:
      cls._destroyMinPermanenceSynapses(connections, random, segment, overrun,
                                        prevWinnerCells)

    # Recalculate in case we weren't able to destroy as many synapses as needed.
    nActual = min(nActual,
                  maxSynapsesPerSegment - connections.numSynapses(segment))

    for _ in range(nActual):
      i = random.getUInt32(len(candidates))
      connections.createSynapse(segment, candidates[i], initialPermanence)
      del candidates[i]


Instruction:
Can you generate the documentation for the following Python 3 function
def _adaptSegment(cls, connections, segment, prevActiveCells,
                    permanenceIncrement, permanenceDecrement):
    """
    Updates synapses on segment.
    Strengthens active synapses; weakens inactive synapses.

    :param connections:          (Object) Connections instance for the tm
    :param segment:              (int)    Segment to adapt
    :param prevActiveCells:      (list)   Active cells in `t-1`
    :param permanenceIncrement:  (float)  Amount to increment active synapses
    :param permanenceDecrement:  (float)  Amount to decrement inactive synapses
    """

    # Destroying a synapse modifies the set that we're iterating through.
    synapsesToDestroy = []

    for synapse in connections.synapsesForSegment(segment):
      permanence = synapse.permanence

      if binSearch(prevActiveCells, synapse.presynapticCell) != -1:
        permanence += permanenceIncrement
      else:
        permanence -= permanenceDecrement

      # Keep permanence within min/max bounds
      permanence = max(0.0, min(1.0, permanence))

      if permanence < EPSILON:
        synapsesToDestroy.append(synapse)
      else:
        connections.updateSynapsePermanence(synapse, permanence)

    for synapse in synapsesToDestroy:
      connections.destroySynapse(synapse)

    if connections.numSynapses(segment) == 0:
      connections.destroySegment(segment)

Response: Adaptes the given segment to the next active synapses and weakens inactive synapses.


Instruction:
Can you create a Python 3 function that
returns the index of the column that a cell belongs to.

Response:
def columnForCell(self, cell):
    """
    Returns the index of the column that a cell belongs to.

    :param cell: (int) Cell index

    :returns: (int) Column index
    """
    self._validateCell(cell)

    return int(cell / self.cellsPerColumn)


Instruction:
Implement a function in Python 3 to
return the indices of cells that belong to a column.

Response:
def cellsForColumn(self, column):
    """
    Returns the indices of cells that belong to a column.

    :param column: (int) Column index

    :returns: (list) Cell indices
    """
    self._validateColumn(column)

    start = self.cellsPerColumn * column
    end = start + self.cellsPerColumn
    return range(start, end)


Instruction:
Can you tell what is the following Python 3 function doing
def mapCellsToColumns(self, cells):
    cellsForColumns = defaultdict(set)

    for cell in cells:
      column = self.columnForCell(cell)
      cellsForColumns[column].add(cell)

    return cellsForColumns

Response: Maps cells to the columns they belong to.


Instruction:
Write a Python 3 script to
return the indices of the predictive cells.

Response:
def getPredictiveCells(self):
    """ Returns the indices of the predictive cells.

    :returns: (list) Indices of predictive cells.
    """
    previousCell = None
    predictiveCells = []
    for segment in self.activeSegments:
      if segment.cell != previousCell:
        predictiveCells.append(segment.cell)
        previousCell = segment.cell

    return predictiveCells


Instruction:
Can you tell what is the following Python 3 function doing
def write(self, proto):
    # capnp fails to save a tuple.  Let's force columnDimensions to list.
    proto.columnDimensions = list(self.columnDimensions)
    proto.cellsPerColumn = self.cellsPerColumn
    proto.activationThreshold = self.activationThreshold
    proto.initialPermanence = round(self.initialPermanence, EPSILON_ROUND)
    proto.connectedPermanence = round(self.connectedPermanence, EPSILON_ROUND)
    proto.minThreshold = self.minThreshold
    proto.maxNewSynapseCount = self.maxNewSynapseCount
    proto.permanenceIncrement = round(self.permanenceIncrement, EPSILON_ROUND)
    proto.permanenceDecrement = round(self.permanenceDecrement, EPSILON_ROUND)
    proto.predictedSegmentDecrement = self.predictedSegmentDecrement

    proto.maxSegmentsPerCell = self.maxSegmentsPerCell
    proto.maxSynapsesPerSegment = self.maxSynapsesPerSegment

    self.connections.write(proto.connections)
    self._random.write(proto.random)

    proto.activeCells = list(self.activeCells)
    proto.winnerCells = list(self.winnerCells)

    protoActiveSegments = proto.init("activeSegments", len(self.activeSegments))
    for i, segment in enumerate(self.activeSegments):
      protoActiveSegments[i].cell = segment.cell
      idx = self.connections.segmentsForCell(segment.cell).index(segment)
      protoActiveSegments[i].idxOnCell = idx

    protoMatchingSegments = proto.init("matchingSegments",
                                       len(self.matchingSegments))
    for i, segment in enumerate(self.matchingSegments):
      protoMatchingSegments[i].cell = segment.cell
      idx = self.connections.segmentsForCell(segment.cell).index(segment)
      protoMatchingSegments[i].idxOnCell = idx

    protoNumActivePotential = proto.init(
      "numActivePotentialSynapsesForSegment",
      len(self.numActivePotentialSynapsesForSegment))
    for i, numActivePotentialSynapses in enumerate(
        self.numActivePotentialSynapsesForSegment):
      segment = self.connections.segmentForFlatIdx(i)
      if segment is not None:
        protoNumActivePotential[i].cell = segment.cell
        idx = self.connections.segmentsForCell(segment.cell).index(segment)
        protoNumActivePotential[i].idxOnCell = idx
        protoNumActivePotential[i].number = numActivePotentialSynapses

    proto.iteration = self.iteration

    protoLastUsedIteration = proto.init(
      "lastUsedIterationForSegment",
      len(self.numActivePotentialSynapsesForSegment))
    for i, lastUsed in enumerate(self.lastUsedIterationForSegment):
      segment = self.connections.segmentForFlatIdx(i)
      if segment is not None:
        protoLastUsedIteration[i].cell = segment.cell
        idx = self.connections.segmentsForCell(segment.cell).index(segment)
        protoLastUsedIteration[i].idxOnCell = idx
        protoLastUsedIteration[i].number = lastUsed

Response: Writes serialized data to proto object.


Instruction:
Can you tell what is the following Python 3 function doing
def read(cls, proto):
    tm = object.__new__(cls)

    # capnp fails to save a tuple, so proto.columnDimensions was forced to
    # serialize as a list.  We prefer a tuple, however, because columnDimensions
    # should be regarded as immutable.
    tm.columnDimensions = tuple(proto.columnDimensions)
    tm.cellsPerColumn = int(proto.cellsPerColumn)
    tm.activationThreshold = int(proto.activationThreshold)
    tm.initialPermanence = round(proto.initialPermanence, EPSILON_ROUND)
    tm.connectedPermanence = round(proto.connectedPermanence, EPSILON_ROUND)
    tm.minThreshold = int(proto.minThreshold)
    tm.maxNewSynapseCount = int(proto.maxNewSynapseCount)
    tm.permanenceIncrement = round(proto.permanenceIncrement, EPSILON_ROUND)
    tm.permanenceDecrement = round(proto.permanenceDecrement, EPSILON_ROUND)
    tm.predictedSegmentDecrement = round(proto.predictedSegmentDecrement,
                                         EPSILON_ROUND)

    tm.maxSegmentsPerCell = int(proto.maxSegmentsPerCell)
    tm.maxSynapsesPerSegment = int(proto.maxSynapsesPerSegment)

    tm.connections = Connections.read(proto.connections)
    #pylint: disable=W0212
    tm._random = Random()
    tm._random.read(proto.random)
    #pylint: enable=W0212

    tm.activeCells = [int(x) for x in proto.activeCells]
    tm.winnerCells = [int(x) for x in proto.winnerCells]

    flatListLength = tm.connections.segmentFlatListLength()
    tm.numActiveConnectedSynapsesForSegment = [0] * flatListLength
    tm.numActivePotentialSynapsesForSegment = [0] * flatListLength
    tm.lastUsedIterationForSegment = [0] * flatListLength

    tm.activeSegments = []
    tm.matchingSegments = []

    for protoSegment in proto.activeSegments:
      tm.activeSegments.append(
        tm.connections.getSegment(protoSegment.cell,
                                  protoSegment.idxOnCell))

    for protoSegment in proto.matchingSegments:
      tm.matchingSegments.append(
        tm.connections.getSegment(protoSegment.cell,
                                  protoSegment.idxOnCell))

    for protoSegment in proto.numActivePotentialSynapsesForSegment:
      segment = tm.connections.getSegment(protoSegment.cell,
                                          protoSegment.idxOnCell)

      tm.numActivePotentialSynapsesForSegment[segment.flatIdx] = (
        int(protoSegment.number))

    tm.iteration = long(proto.iteration)

    for protoSegment in proto.lastUsedIterationForSegment:
      segment = tm.connections.getSegment(protoSegment.cell,
                                          protoSegment.idxOnCell)

      tm.lastUsedIterationForSegment[segment.flatIdx] = (
        long(protoSegment.number))

    return tm

Response: Reads deserialized data from proto object.


Instruction:
Given the following Python 3 function, write the documentation
def generateFromNumbers(self, numbers):
    sequence = []

    for number in numbers:
      if number == None:
        sequence.append(number)
      else:
        pattern = self.patternMachine.get(number)
        sequence.append(pattern)

    return sequence

Response: Generate a sequence from a list of numbers.


Instruction:
How would you code a function in Python 3 to
add spatial noise to each pattern in the sequence.

Response:
def addSpatialNoise(self, sequence, amount):
    """
    Add spatial noise to each pattern in the sequence.

    @param sequence (list)  Sequence
    @param amount   (float) Amount of spatial noise

    @return (list) Sequence with spatial noise
    """
    newSequence = []

    for pattern in sequence:
      if pattern is not None:
        pattern = self.patternMachine.addNoise(pattern, amount)
      newSequence.append(pattern)

    return newSequence


Instruction:
Create a Python 3 function to
compute ROC curve for a Receiver operating characteristic.

Response:
def ROCCurve(y_true, y_score):
    """compute Receiver operating characteristic (ROC)

    Note: this implementation is restricted to the binary classification task.

    Parameters
    ----------

    y_true : array, shape = [n_samples]
        true binary labels

    y_score : array, shape = [n_samples]
        target scores, can either be probability estimates of
        the positive class, confidence values, or binary decisions.

    Returns
    -------
    fpr : array, shape = [>2]
        False Positive Rates

    tpr : array, shape = [>2]
        True Positive Rates

    thresholds : array, shape = [>2]
        Thresholds on y_score used to compute fpr and tpr

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn import metrics
    >>> y = np.array([1, 1, 2, 2])
    >>> scores = np.array([0.1, 0.4, 0.35, 0.8])
    >>> fpr, tpr, thresholds = metrics.roc_curve(y, scores)
    >>> fpr
    array([ 0. ,  0.5,  0.5,  1. ])

    References
    ----------
    http://en.wikipedia.org/wiki/Receiver_operating_characteristic

    """
    y_true = np.ravel(y_true)
    classes = np.unique(y_true)

    # ROC only for binary classification
    if classes.shape[0] != 2:
        raise ValueError("ROC is defined for binary classification only")

    y_score = np.ravel(y_score)

    n_pos = float(np.sum(y_true == classes[1]))  # nb of true positive
    n_neg = float(np.sum(y_true == classes[0]))  # nb of true negative

    thresholds = np.unique(y_score)
    neg_value, pos_value = classes[0], classes[1]

    tpr = np.empty(thresholds.size, dtype=np.float)  # True positive rate
    fpr = np.empty(thresholds.size, dtype=np.float)  # False positive rate

    # Build tpr/fpr vector
    current_pos_count = current_neg_count = sum_pos = sum_neg = idx = 0

    signal = np.c_[y_score, y_true]
    sorted_signal = signal[signal[:, 0].argsort(), :][::-1]
    last_score = sorted_signal[0][0]
    for score, value in sorted_signal:
        if score == last_score:
            if value == pos_value:
                current_pos_count += 1
            else:
                current_neg_count += 1
        else:
            tpr[idx] = (sum_pos + current_pos_count) / n_pos
            fpr[idx] = (sum_neg + current_neg_count) / n_neg
            sum_pos += current_pos_count
            sum_neg += current_neg_count
            current_pos_count = 1 if value == pos_value else 0
            current_neg_count = 1 if value == neg_value else 0
            idx += 1
            last_score = score
    else:
        tpr[-1] = (sum_pos + current_pos_count) / n_pos
        fpr[-1] = (sum_neg + current_neg_count) / n_neg

    # hard decisions, add (0,0)
    if fpr.shape[0] == 2:
        fpr = np.array([0.0, fpr[0], fpr[1]])
        tpr = np.array([0.0, tpr[0], tpr[1]])
    # trivial decisions, add (0,0) and (1,1)
    elif fpr.shape[0] == 1:
        fpr = np.array([0.0, fpr[0], 1.0])
        tpr = np.array([0.0, tpr[0], 1.0])

    return fpr, tpr, thresholds


Instruction:
Write a Python 3 function that can
compute the area under curve using the trapezoidal rule

Response:
def AreaUnderCurve(x, y):
    """Compute Area Under the Curve (AUC) using the trapezoidal rule

    Parameters
    ----------
    x : array, shape = [n]
        x coordinates

    y : array, shape = [n]
        y coordinates

    Returns
    -------
    auc : float

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn import metrics
    >>> y = np.array([1, 1, 2, 2])
    >>> pred = np.array([0.1, 0.4, 0.35, 0.8])
    >>> fpr, tpr, thresholds = metrics.roc_curve(y, pred)
    >>> metrics.auc(fpr, tpr)
    0.75

    """
    #x, y = check_arrays(x, y)
    if x.shape[0] != y.shape[0]:
        raise ValueError('x and y should have the same shape'
                         ' to compute area under curve,'
                         ' but x.shape = %s and y.shape = %s.'
                         % (x.shape, y.shape))
    if x.shape[0] < 2:
        raise ValueError('At least 2 points are needed to compute'
                         ' area under curve, but x.shape = %s' % x.shape)

    # reorder the data points according to the x axis
    order = np.argsort(x)
    x = x[order]
    y = y[order]

    h = np.diff(x)
    area = np.sum(h * (y[1:] + y[:-1])) / 2.0
    return area


Instruction:
Can you tell what is the following Python 3 function doing
def mmPrettyPrintTraces(traces, breakOnResets=None):
    assert len(traces) > 0, "No traces found"
    table = PrettyTable(["#"] + [trace.prettyPrintTitle() for trace in traces])

    for i in xrange(len(traces[0].data)):
      if breakOnResets and breakOnResets.data[i]:
        table.add_row(["<reset>"] * (len(traces) + 1))
      table.add_row([i] +
        [trace.prettyPrintDatum(trace.data[i]) for trace in traces])

    return table.get_string().encode("utf-8")

Response: Pretty - print a list of traces in a node.


Instruction:
Create a Python 3 function to
return pretty-printed table of metrics. @param metrics (list) Traces to print in table @param sigFigs (int) Number of significant figures to print @return (string) Pretty-printed table of metrics.

Response:
def mmPrettyPrintMetrics(metrics, sigFigs=5):
    """
    Returns pretty-printed table of metrics.

    @param metrics (list) Traces to print in table
    @param sigFigs (int)  Number of significant figures to print

    @return (string) Pretty-printed table of metrics.
    """
    assert len(metrics) > 0, "No metrics found"
    table = PrettyTable(["Metric", "mean", "standard deviation",
                         "min", "max", "sum", ])

    for metric in metrics:
      table.add_row([metric.prettyPrintTitle()] + metric.getStats())

    return table.get_string().encode("utf-8")


Instruction:
Given the following Python 3 function, write the documentation
def mmGetCellTracePlot(self, cellTrace, cellCount, activityType, title="",
                         showReset=False, resetShading=0.25):
    """
    Returns plot of the cell activity. Note that if many timesteps of
    activities are input, matplotlib's image interpolation may omit activities
    (columns in the image).

    @param cellTrace    (list)   a temporally ordered list of sets of cell
                                 activities

    @param cellCount    (int)    number of cells in the space being rendered

    @param activityType (string) type of cell activity being displayed

    @param title        (string) an optional title for the figure

    @param showReset    (bool)   if true, the first set of cell activities
                                 after a reset will have a grayscale background

    @param resetShading (float)  applicable if showReset is true, specifies the
                                 intensity of the reset background with 0.0
                                 being white and 1.0 being black

    @return (Plot) plot
    """
    plot = Plot(self, title)
    resetTrace = self.mmGetTraceResets().data
    data = numpy.zeros((cellCount, 1))
    for i in xrange(len(cellTrace)):
      # Set up a "background" vector that is shaded or blank
      if showReset and resetTrace[i]:
        activity = numpy.ones((cellCount, 1)) * resetShading
      else:
        activity = numpy.zeros((cellCount, 1))

      activeIndices = cellTrace[i]
      activity[list(activeIndices)] = 1
      data = numpy.concatenate((data, activity), 1)

    plot.add2DArray(data, xlabel="Time", ylabel=activityType, name=title)
    return plot

Response: Returns a matplotlib plot of the cell activity.