Instruction:
Can you generate a brief explanation for the following Python 3 code
def spDiff(SP1,SP2):
    if(len(SP1._masterConnectedM)!=len(SP2._masterConnectedM)):
        print "Connected synapse matrices are different sizes"
        return False

    if(len(SP1._masterPotentialM)!=len(SP2._masterPotentialM)):
        print "Potential synapse matrices are different sizes"
        return False

    if(len(SP1._masterPermanenceM)!=len(SP2._masterPermanenceM)):
        print "Permanence matrices are different sizes"
        return False

    #iterate over cells
    for i in range(0,len(SP1._masterConnectedM)):
        #grab the Coincidence Matrices and compare them
        connected1 = SP1._masterConnectedM[i]
        connected2 = SP2._masterConnectedM[i]
        if(connected1!=connected2):
            print "Connected Matrices for cell %d different"  % (i)
            return False
        #grab permanence Matrices and compare them
        permanences1 = SP1._masterPermanenceM[i];
        permanences2 = SP2._masterPermanenceM[i];
        if(permanences1!=permanences2):
            print "Permanence Matrices for cell %d different" % (i)
            return False
        #grab the potential connection Matrices and compare them
        potential1 = SP1._masterPotentialM[i];
        potential2 = SP2._masterPotentialM[i];
        if(potential1!=potential2):
            print "Potential Matrices for cell %d different" % (i)
            return False

    #Check firing boosts
    if(not numpy.array_equal(SP1._firingBoostFactors,SP2._firingBoostFactors)):
        print "Firing boost factors are different between spatial poolers"
        return False

    #Check duty cycles after inhibiton
    if(not numpy.array_equal(SP1._dutyCycleAfterInh,SP2._dutyCycleAfterInh)):
        print "Duty cycles after inhibition are different between spatial poolers"
        return False

    #Check duty cycles before inhibition
    if(not numpy.array_equal(SP1._dutyCycleBeforeInh,SP2._dutyCycleBeforeInh)):
        print "Duty cycles before inhibition are different between spatial poolers"
        return False

    print("Spatial Poolers are equivalent")

    return True

Response: This function compares two spatial pooler instances and returns True if the two spatial pooler instances are equivalent False otherwise.


Instruction:
Make a summary of the following Python 3 code
def removeSeqStarts(vectors, resets, numSteps=1):

  # Do nothing if numSteps is 0
  if numSteps == 0:
    return vectors

  resetIndices = resets.nonzero()[0]
  removeRows = resetIndices
  for i in range(numSteps-1):
    removeRows = numpy.hstack((removeRows, resetIndices+i+1))

  return numpy.delete(vectors, removeRows, axis=0)

Response: This function removes the first numSteps samples from the start of each sequence in the sequence.


Instruction:
Can you generate the documentation for the following Python 3 function
def _accumulateFrequencyCounts(values, freqCounts=None):

  # How big does our freqCounts vector need to be?
  values = numpy.array(values)
  numEntries = values.max() + 1
  if freqCounts is not None:
    numEntries = max(numEntries, freqCounts.size)

  # Where do we accumulate the results?
  if freqCounts is not None:
    if freqCounts.size != numEntries:
      newCounts = numpy.zeros(numEntries, dtype='int32')
      newCounts[0:freqCounts.size] = freqCounts
    else:
      newCounts = freqCounts
  else:
    newCounts = numpy.zeros(numEntries, dtype='int32')

  # Accumulate the new values
  for v in values:
    newCounts[v] += 1

  return newCounts

Response: This function is used to accumulate a list of values into the frequency counts and return the updated frequency counts.


Instruction:
Write a Python 3 function that can
return 3 things for a vector.

Response:
def _listOfOnTimesInVec(vector):
  """
  Returns 3 things for a vector:
    * the total on time
    * the number of runs
    * a list of the durations of each run.

  Parameters:
  -----------------------------------------------
  input stream: 11100000001100000000011111100000
  return value: (11, 3, [3, 2, 6])
  """

  # init counters
  durations = []
  numOnTimes   = 0
  totalOnTime = 0

  # Find where the nonzeros are
  nonzeros = numpy.array(vector).nonzero()[0]

  # Nothing to do if vector is empty
  if len(nonzeros) == 0:
    return (0, 0, [])

  # Special case of only 1 on bit
  if len(nonzeros) == 1:
    return (1, 1, [1])

  # Count the consecutive non-zeros
  prev = nonzeros[0]
  onTime = 1
  endIdx = nonzeros[-1]
  for idx in nonzeros[1:]:
    if idx != prev+1:
      totalOnTime += onTime
      numOnTimes  += 1
      durations.append(onTime)
      onTime       = 1
    else:
      onTime += 1
    prev = idx

  # Add in the last one
  totalOnTime += onTime
  numOnTimes  += 1
  durations.append(onTime)

  return (totalOnTime, numOnTimes, durations)


Instruction:
Given the following Python 3 function, write the documentation
def averageOnTimePerTimestep(vectors, numSamples=None):

  # Special case given a 1 dimensional vector: it represents a single column
  if vectors.ndim == 1:
    vectors.shape = (-1,1)
  numTimeSteps = len(vectors)
  numElements  = len(vectors[0])

  # How many samples will we look at?
  if numSamples is not None:
    import pdb; pdb.set_trace()   # Test this....
    countOn    = numpy.random.randint(0, numElements, numSamples)
    vectors = vectors[:, countOn]

  # Fill in each non-zero of vectors with the on-time that that output was
  #  on for.
  durations = numpy.zeros(vectors.shape, dtype='int32')
  for col in xrange(vectors.shape[1]):
    _fillInOnTimes(vectors[:,col], durations[:,col])

  # Compute the average on time for each time step
  sums = vectors.sum(axis=1)
  sums.clip(min=1, max=numpy.inf, out=sums)
  avgDurations = durations.sum(axis=1, dtype='float64') / sums
  avgOnTime = avgDurations.sum() / (avgDurations > 0).sum()

  # Generate the frequency counts for each duration
  freqCounts = _accumulateFrequencyCounts(avgDurations)
  return (avgOnTime, freqCounts)

Response: Calculates the average on - time over all time steps and calculates the average on - time over all time steps.


Instruction:
Write a Python 3 function for
returning the average on - time of all outputs.

Response:
def averageOnTime(vectors, numSamples=None):
  """
  Returns the average on-time, averaged over all on-time runs.

  Parameters:
  -----------------------------------------------
  vectors:          the vectors for which the onTime is calculated. Row 0
                    contains the outputs from time step 0, row 1 from time step
                    1, etc.
  numSamples:       the number of elements for which on-time is calculated.
                    If not specified, then all elements are looked at.

  Returns:    (scalar average on-time of all outputs,
               list containing frequency counts of each encountered on-time)

  """

  # Special case given a 1 dimensional vector: it represents a single column
  if vectors.ndim == 1:
    vectors.shape = (-1,1)
  numTimeSteps = len(vectors)
  numElements  = len(vectors[0])

  # How many samples will we look at?
  if numSamples is None:
    numSamples = numElements
    countOn    = range(numElements)
  else:
    countOn    = numpy.random.randint(0, numElements, numSamples)

  # Compute the on-times and accumulate the frequency counts of each on-time
  #  encountered
  sumOfLengths = 0.0
  onTimeFreqCounts = None
  n = 0
  for i in countOn:
    (onTime, segments, durations) = _listOfOnTimesInVec(vectors[:,i])
    if onTime != 0.0:
      sumOfLengths += onTime
      n += segments
      onTimeFreqCounts = _accumulateFrequencyCounts(durations, onTimeFreqCounts)

  # Return the average on time of each element that was on.
  if n > 0:
    return (sumOfLengths/n, onTimeFreqCounts)
  else:
    return (0.0, onTimeFreqCounts)


Instruction:
Given the following Python 3 function, write the documentation
def plotOutputsOverTime(vectors, buVectors=None, title='On-times'):

  # Produce the plot
  import pylab
  pylab.ion()
  pylab.figure()
  imData = vectors.transpose()
  if buVectors is not None:
    assert(buVectors.shape == vectors.shape)
    imData = imData.copy()
    imData[buVectors.transpose().astype('bool')] = 2
  pylab.imshow(imData, aspect='auto', cmap=pylab.cm.gray_r,
                interpolation='nearest')

  pylab.title(title)

Response: Generates a figure that shows each output of the sequence over time.


Instruction:
Make a summary of the following Python 3 code
def plotHistogram(freqCounts, title='On-Times Histogram', xLabel='On-Time'):

  import pylab
  pylab.ion()
  pylab.figure()
  pylab.bar(numpy.arange(len(freqCounts)) - 0.5, freqCounts)
  pylab.title(title)
  pylab.xlabel(xLabel)

Response: Plots a histogram of the frequency counts of the on - times encountered by the user.


Instruction:
Create a Python 3 function to
calculate the stability of the active elements in a population over multiple time steps.

Response:
def populationStability(vectors, numSamples=None):
  """
  Returns the stability for the population averaged over multiple time steps

  Parameters:
  -----------------------------------------------
  vectors:          the vectors for which the stability is calculated
  numSamples        the number of time steps where stability is counted

  At each time step, count the fraction of the active elements which are stable
  from the previous step
  Average all the fraction

  """

  # ----------------------------------------------------------------------
  # Calculate the stability
  numVectors = len(vectors)

  if numSamples is None:
    numSamples = numVectors-1
    countOn = range(numVectors-1)
  else:
    countOn = numpy.random.randint(0, numVectors-1, numSamples)

  sigmap = 0.0
  for i in countOn:
    match = checkMatch(vectors[i], vectors[i+1], sparse=False)
    # Ignore reset vectors (all 0's)
    if match[1] != 0:
      sigmap += float(match[0])/match[1]

  return sigmap / numSamples


Instruction:
Create a Python 3 function to
return the percent of the outputs that remain completely stable over N time steps.

Response:
def percentOutputsStableOverNTimeSteps(vectors, numSamples=None):
  """
  Returns the percent of the outputs that remain completely stable over
  N time steps.

  Parameters:
  -----------------------------------------------
  vectors:        the vectors for which the stability is calculated
  numSamples:     the number of time steps where stability is counted

  For each window of numSamples, count how many outputs are active during
  the entire window.

  """

  # ----------------------------------------------------------------------
  # Calculate the stability
  totalSamples = len(vectors)
  windowSize = numSamples

  # Process each window
  numWindows = 0
  pctStable = 0

  for wStart in range(0, totalSamples-windowSize+1):
    # Count how many elements are active for the entire time
    data = vectors[wStart:wStart+windowSize]
    outputSums = data.sum(axis=0)
    stableOutputs = (outputSums == windowSize).sum()

    # Accumulated
    samplePctStable = float(stableOutputs) / data[0].sum()
    print samplePctStable
    pctStable += samplePctStable
    numWindows += 1

  # Return percent average over all possible windows
  return float(pctStable) / numWindows


Instruction:
Create a Python 3 function to
compute the saturation for a continuous level. This breaks the level into multiple regions and computes the saturation level for each region. Parameters: -------------------------------------------- outputs: output of the level. If sparseForm is True, this is a list of the non-zeros. If sparseForm is False, it is the dense representation outputsShape: The shape of the outputs of the level (height, width) retval: (sat, innerSat): sat: list of the saturation levels of each non-empty region of the level (each 0 -> 1.0) innerSat: list of the saturation level of each non-empty region that is not near an edge (each 0 -> 1.0)

Response:
def computeSaturationLevels(outputs, outputsShape, sparseForm=False):
  """
  Compute the saturation for a continuous level. This breaks the level into
  multiple regions and computes the saturation level for each region.

  Parameters:
  --------------------------------------------
  outputs:      output of the level. If sparseForm is True, this is a list of
                  the non-zeros. If sparseForm is False, it is the dense
                  representation
  outputsShape: The shape of the outputs of the level (height, width)
  retval:       (sat, innerSat):
                  sat: list of the saturation levels of each non-empty
                    region of the level (each 0 -> 1.0)
                  innerSat: list of the saturation level of each non-empty region
                       that is not near an edge (each 0 -> 1.0)

  """

  # Get the outputs into a SparseBinaryMatrix
  if not sparseForm:
    outputs = outputs.reshape(outputsShape)
    spOut = SM32(outputs)
  else:
    if len(outputs) > 0:
      assert (outputs.max() < outputsShape[0] * outputsShape[1])
    spOut = SM32(1, outputsShape[0] * outputsShape[1])
    spOut.setRowFromSparse(0, outputs, [1]*len(outputs))
    spOut.reshape(outputsShape[0], outputsShape[1])

  # Get the activity in each local region using the nNonZerosPerBox method
  # This method takes a list of the end row indices and a list of the end
  #  column indices.
  # We will use regions that are 15x15, which give us about a 1/225 (.4%) resolution
  #  on saturation.
  regionSize = 15
  rows = xrange(regionSize+1, outputsShape[0]+1, regionSize)
  cols = xrange(regionSize+1, outputsShape[1]+1, regionSize)
  regionSums = spOut.nNonZerosPerBox(rows, cols)

  # Get all the nonzeros out - those are our saturation sums
  (locations, values) = regionSums.tolist()
  values /= float(regionSize * regionSize)
  sat = list(values)

  # Now, to compute which are the inner regions, we will only take the ones that
  #  are surrounded by activity above, below, left and right
  innerSat = []
  locationSet = set(locations)
  for (location, value) in itertools.izip(locations, values):
    (row, col) = location
    if (row-1,col) in locationSet and (row, col-1) in locationSet \
      and (row+1, col) in locationSet and (row, col+1) in locationSet:
      innerSat.append(value)

  return (sat, innerSat)


Instruction:
How would you explain what the following Python 3 function does
def checkMatch(input, prediction, sparse=True, verbosity=0):

  if sparse:
    activeElementsInInput = set(input)
    activeElementsInPrediction = set(prediction)

  else:
    activeElementsInInput = set(input.nonzero()[0])
    activeElementsInPrediction = set(prediction.nonzero()[0])

  totalActiveInPrediction = len(activeElementsInPrediction)
  totalActiveInInput     = len(activeElementsInInput)

  foundInInput = len(activeElementsInPrediction.intersection(activeElementsInInput))
  missingFromInput = len(activeElementsInPrediction.difference(activeElementsInInput))
  missingFromPrediction = len(activeElementsInInput.difference(activeElementsInPrediction))

  if verbosity >= 1:
    print "preds. found in input:", foundInInput, "out of", totalActiveInPrediction,
    print "; preds. missing from input:", missingFromInput, "out of", \
              totalActiveInPrediction,
    print "; unexpected active in input:", missingFromPrediction, "out of", \
              totalActiveInInput

  return (foundInInput, totalActiveInInput, missingFromInput,
          totalActiveInPrediction)

Response: Checks the input and prediction for the actual input and returns the result.


Instruction:
Write a Python 3 script for
computing the predictive ability of a temporal memory (TM). This routine returns a value which is the average number of time steps of prediction provided by the TM. It accepts as input the inputs, outputs, and resets provided to the TM as well as a 'minOverlapPct' used to evalulate whether or not a prediction is a good enough match to the actual input. The 'outputs' are the pooling outputs of the TM. This routine treats each output as a "manifold" that includes the active columns that should be present in the next N inputs. It then looks at each successive input and sees if it's active columns are within the manifold. For each output sample, it computes how many time steps it can go forward on the input before the input overlap with the manifold is less then 'minOverlapPct'. It returns the average number of time steps calculated for each output. Parameters: ----------------------------------------------- inputs: The inputs to the TM. Row 0 contains the inputs from time step 0, row 1 from time step 1, etc. resets: The reset input to the TM. Element 0 contains the reset from time step 0, element 1 from time step 1, etc. outputs: The pooling outputs from the TM. Row 0 contains the outputs from time step 0, row 1 from time step 1, etc. minOverlapPct: How much each input's columns must overlap with the pooling output's columns to be considered a valid prediction. retval: (Average number of time steps of prediction over all output samples, Average number of time steps of prediction when we aren't cut short by the end of the sequence, List containing frequency counts of each encountered prediction time)

Response:
def predictionExtent(inputs, resets, outputs, minOverlapPct=100.0):
  """
  Computes the predictive ability of a temporal memory (TM). This routine returns
  a value which is the average number of time steps of prediction provided
  by the TM. It accepts as input the inputs, outputs, and resets provided to
  the TM as well as a 'minOverlapPct' used to evalulate whether or not a
  prediction is a good enough match to the actual input.

  The 'outputs' are the pooling outputs of the TM. This routine treats each output
  as a "manifold" that includes the active columns that should be present in the
  next N inputs. It then looks at each successive input and sees if it's active
  columns are within the manifold. For each output sample, it computes how
  many time steps it can go forward on the input before the input overlap with
  the manifold is less then 'minOverlapPct'. It returns the average number of
  time steps calculated for each output.

  Parameters:
  -----------------------------------------------
  inputs:          The inputs to the TM. Row 0 contains the inputs from time
                   step 0, row 1 from time step 1, etc.
  resets:          The reset input to the TM. Element 0 contains the reset from
                   time step 0, element 1 from time step 1, etc.
  outputs:         The pooling outputs from the TM. Row 0 contains the outputs
                   from time step 0, row 1 from time step 1, etc.
  minOverlapPct:   How much each input's columns must overlap with the pooling
                   output's columns to be considered a valid prediction.

  retval:          (Average number of time steps of prediction over all output
                     samples,
                    Average number of time steps of prediction when we aren't
                     cut short by the end of the sequence,
                    List containing frequency counts of each encountered
                     prediction time)

  """

  # List of how many times we encountered each prediction amount. Element 0
  #  is how many times we successfully predicted 0 steps in advance, element 1
  #  is how many times we predicted 1 step in advance, etc.
  predCounts = None

  # Total steps of prediction over all samples
  predTotal = 0

  # Total number of samples
  nSamples = len(outputs)

  # Total steps of prediction for samples at the start of the sequence, or
  #  for samples whose prediction runs aren't cut short by the end of the
  #  sequence.
  predTotalNotLimited = 0
  nSamplesNotLimited = 0

  # Compute how many cells/column we have
  nCols = len(inputs[0])
  nCellsPerCol = len(outputs[0]) // nCols

  # Evalulate prediction for each output sample
  for idx in xrange(nSamples):

    # What are the active columns for this output?
    activeCols = outputs[idx].reshape(nCols, nCellsPerCol).max(axis=1)

    # How many steps of prediction do we have?
    steps = 0
    while (idx+steps+1 < nSamples) and (resets[idx+steps+1] == 0):
      overlap = numpy.logical_and(inputs[idx+steps+1], activeCols)
      overlapPct = 100.0 * float(overlap.sum()) / inputs[idx+steps+1].sum()
      if overlapPct >= minOverlapPct:
        steps += 1
      else:
        break

    # print "idx:", idx, "steps:", steps
    # Accumulate into our total
    predCounts = _accumulateFrequencyCounts([steps], predCounts)
    predTotal += steps

    # If this sample was not cut short by the end of the sequence, include
    #  it into the "NotLimited" runs
    if resets[idx] or \
        ((idx+steps+1 < nSamples) and (not resets[idx+steps+1])):
      predTotalNotLimited += steps
      nSamplesNotLimited += 1

  # Return results
  return (float(predTotal) / nSamples,
          float(predTotalNotLimited) / nSamplesNotLimited,
          predCounts)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def getCentreAndSpreadOffsets(spaceShape,
                              spreadShape,
                              stepSize=1):
  """
  Generates centre offsets and spread offsets for block-mode based training
  regimes - star, cross, block.

    Parameters:
    -----------------------------------------------
    spaceShape:   The (height, width) of the 2-D space to explore. This
                  sets the number of center-points.
    spreadShape:  The shape (height, width) of the area around each center-point
                  to explore.
    stepSize:     The step size. How big each step is, in pixels. This controls
                  *both* the spacing of the center-points within the block and the
                  points we explore around each center-point
    retval:       (centreOffsets, spreadOffsets)
  """

  from nupic.math.cross import cross
  # =====================================================================
  # Init data structures
  # What is the range on the X and Y offsets of the center points?
  shape = spaceShape
  # If the shape is (1,1), special case of just 1 center point
  if shape[0] == 1 and shape[1] == 1:
    centerOffsets = [(0,0)]
  else:
    xMin = -1 * (shape[1] // 2)
    xMax = xMin + shape[1] - 1
    xPositions = range(stepSize * xMin, stepSize * xMax + 1, stepSize)

    yMin = -1 * (shape[0] // 2)
    yMax = yMin + shape[0] - 1
    yPositions = range(stepSize * yMin, stepSize * yMax + 1, stepSize)

    centerOffsets = list(cross(yPositions, xPositions))

  numCenterOffsets = len(centerOffsets)
  print "centerOffsets:", centerOffsets

  # What is the range on the X and Y offsets of the spread points?
  shape = spreadShape
  # If the shape is (1,1), special case of no spreading around each center
  #  point
  if shape[0] == 1 and shape[1] == 1:
    spreadOffsets = [(0,0)]
  else:
    xMin = -1 * (shape[1] // 2)
    xMax = xMin + shape[1] - 1
    xPositions = range(stepSize * xMin, stepSize * xMax + 1, stepSize)

    yMin = -1 * (shape[0] // 2)
    yMax = yMin + shape[0] - 1
    yPositions = range(stepSize * yMin, stepSize * yMax + 1, stepSize)

    spreadOffsets = list(cross(yPositions, xPositions))

    # Put the (0,0) entry first
    spreadOffsets.remove((0,0))
    spreadOffsets.insert(0, (0,0))

  numSpreadOffsets = len(spreadOffsets)
  print "spreadOffsets:", spreadOffsets

  return centerOffsets, spreadOffsets

Response: This function generates centre offsets and spread offsets for a single training block.


Instruction:
Explain what the following Python 3 code does
def makeCloneMap(columnsShape, outputCloningWidth, outputCloningHeight=-1):
  if outputCloningHeight < 0:
    outputCloningHeight = outputCloningWidth

  columnsHeight, columnsWidth = columnsShape

  numDistinctMasters = outputCloningWidth * outputCloningHeight

  a = numpy.empty((columnsHeight, columnsWidth), 'uint32')
  for row in xrange(columnsHeight):
    for col in xrange(columnsWidth):
      a[row, col] = (col % outputCloningWidth) + \
                    (row % outputCloningHeight) * outputCloningWidth

  return a, numDistinctMasters

Response: Make a two - dimensional clone map mapping columns to clone master.


Instruction:
Here you have a function in Python 3, explain what it does
def numpyStr(array, format='%f', includeIndices=False, includeZeros=True):

  shape = array.shape
  assert (len(shape) <= 2)
  items = ['[']
  if len(shape) == 1:
    if includeIndices:
      format = '%d:' + format
      if includeZeros:
        rowItems = [format % (c,x) for (c,x) in enumerate(array)]
      else:
        rowItems = [format % (c,x) for (c,x) in enumerate(array) if x != 0]
    else:
      rowItems = [format % (x) for x in array]
    items.extend(rowItems)

  else:
    (rows, cols) = shape
    if includeIndices:
      format = '%d,%d:' + format

    for r in xrange(rows):
      if includeIndices:
        rowItems = [format % (r,c,x) for c,x in enumerate(array[r])]
      else:
        rowItems = [format % (x) for x in array[r]]
      if r > 0:
        items.append('')

      items.append('[')
      items.extend(rowItems)
      if r < rows-1:
        items.append(']\n')
      else:
        items.append(']')

  items.append(']')
  return ' '.join(items)

Response: Pretty print a numpy array using the given format string for each value.


Instruction:
Implement a function in Python 3 to
generate a random sample from the discrete probability distribution and returns its value and the log of the probability of sampling that value.

Response:
def sample(self, rgen):
    """Generates a random sample from the discrete probability distribution
    and returns its value and the log of the probability of sampling that value.
    """
    rf = rgen.uniform(0, self.sum)
    index = bisect.bisect(self.cdf, rf)
    return self.keys[index], numpy.log(self.pmf[index])


Instruction:
Implement a Python 3 function for
logging probability of distribution.

Response:
def logProbability(self, distn):
    """Form of distribution must be an array of counts in order of self.keys."""
    x = numpy.asarray(distn)
    n = x.sum()
    return (logFactorial(n) - numpy.sum([logFactorial(k) for k in x]) +
      numpy.sum(x * numpy.log(self.dist.pmf)))


Instruction:
How would you explain what the following Python 3 function does
def sample(self, rgen):
    x = rgen.poisson(self.lambdaParameter)
    return x, self.logDensity(x)

Response:
Generates a random sample from the Poisson probability distribution and
    returns its value and the log of the probability of sampling that value.


Instruction:
Write a Python 3 function that can
link sensor region to other region so that it can pass it data.

Response:
def createDataOutLink(network, sensorRegionName, regionName):
  """Link sensor region to other region so that it can pass it data."""
  network.link(sensorRegionName, regionName, "UniformLink", "",
               srcOutput="dataOut", destInput="bottomUpIn")


Instruction:
Write a Python 3 script to
create a feed - forward link between 2 regions.

Response:
def createFeedForwardLink(network, regionName1, regionName2):
  """Create a feed-forward link between 2 regions: regionName1 -> regionName2"""
  network.link(regionName1, regionName2, "UniformLink", "",
               srcOutput="bottomUpOut", destInput="bottomUpIn")


Instruction:
How would you implement a function in Python 3 that
creates a reset link from a sensor region.

Response:
def createResetLink(network, sensorRegionName, regionName):
  """Create a reset link from a sensor region: sensorRegionName -> regionName"""
  network.link(sensorRegionName, regionName, "UniformLink", "",
               srcOutput="resetOut", destInput="resetIn")


Instruction:
Create a Python 3 function for
creating required links from a sensor region to a classifier region.

Response:
def createSensorToClassifierLinks(network, sensorRegionName,
                                  classifierRegionName):
  """Create required links from a sensor region to a classifier region."""
  network.link(sensorRegionName, classifierRegionName, "UniformLink", "",
               srcOutput="bucketIdxOut", destInput="bucketIdxIn")
  network.link(sensorRegionName, classifierRegionName, "UniformLink", "",
               srcOutput="actValueOut", destInput="actValueIn")
  network.link(sensorRegionName, classifierRegionName, "UniformLink", "",
               srcOutput="categoryOut", destInput="categoryIn")


Instruction:
Can you generate a brief explanation for the following Python 3 code
def createNetwork(dataSource):
  with open(_PARAMS_PATH, "r") as f:
    modelParams = yaml.safe_load(f)["modelParams"]

  # Create a network that will hold the regions.
  network = Network()

  # Add a sensor region.
  network.addRegion("sensor", "py.RecordSensor", '{}')

  # Set the encoder and data source of the sensor region.
  sensorRegion = network.regions["sensor"].getSelf()
  sensorRegion.encoder = createEncoder(modelParams["sensorParams"]["encoders"])
  sensorRegion.dataSource = dataSource

  # Make sure the SP input width matches the sensor region output width.
  modelParams["spParams"]["inputWidth"] = sensorRegion.encoder.getWidth()

  # Add SP and TM regions.
  network.addRegion("SP", "py.SPRegion", json.dumps(modelParams["spParams"]))
  network.addRegion("TM", "py.TMRegion", json.dumps(modelParams["tmParams"]))

  # Add a classifier region.
  clName = "py.%s" % modelParams["clParams"].pop("regionName")
  network.addRegion("classifier", clName, json.dumps(modelParams["clParams"]))

  # Add all links
  createSensorToClassifierLinks(network, "sensor", "classifier")
  createDataOutLink(network, "sensor", "SP")
  createFeedForwardLink(network, "SP", "TM")
  createFeedForwardLink(network, "TM", "classifier")
  # Reset links are optional, since the sensor region does not send resets.
  createResetLink(network, "sensor", "SP")
  createResetLink(network, "sensor", "TM")

  # Make sure all objects are initialized.
  network.initialize()

  return network

Response: Create and initialize a network.


Instruction:
Explain what the following Python 3 code does
def getPredictionResults(network, clRegionName):
  classifierRegion = network.regions[clRegionName]
  actualValues = classifierRegion.getOutputData("actualValues")
  probabilities = classifierRegion.getOutputData("probabilities")
  steps = classifierRegion.getSelf().stepsList
  N = classifierRegion.getSelf().maxCategoryCount
  results = {step: {} for step in steps}
  for i in range(len(steps)):
    # stepProbabilities are probabilities for this prediction step only.
    stepProbabilities = probabilities[i * N:(i + 1) * N - 1]
    mostLikelyCategoryIdx = stepProbabilities.argmax()
    predictedValue = actualValues[mostLikelyCategoryIdx]
    predictionConfidence = stepProbabilities[mostLikelyCategoryIdx]
    results[steps[i]]["predictedValue"] = predictedValue
    results[steps[i]]["predictionConfidence"] = predictionConfidence
  return results

Response: Get prediction results for all prediction steps.


Instruction:
Can you implement a function in Python 3 that
runs the Hot Gym example.

Response:
def runHotgym(numRecords):
  """Run the Hot Gym example."""

  # Create a data source for the network.
  dataSource = FileRecordStream(streamID=_INPUT_FILE_PATH)
  numRecords = min(numRecords, dataSource.getDataRowCount())
  network = createNetwork(dataSource)

  # Set predicted field
  network.regions["sensor"].setParameter("predictedField", "consumption")

  # Enable learning for all regions.
  network.regions["SP"].setParameter("learningMode", 1)
  network.regions["TM"].setParameter("learningMode", 1)
  network.regions["classifier"].setParameter("learningMode", 1)

  # Enable inference for all regions.
  network.regions["SP"].setParameter("inferenceMode", 1)
  network.regions["TM"].setParameter("inferenceMode", 1)
  network.regions["classifier"].setParameter("inferenceMode", 1)

  results = []
  N = 1  # Run the network, N iterations at a time.
  for iteration in range(0, numRecords, N):
    network.run(N)

    predictionResults = getPredictionResults(network, "classifier")
    oneStep = predictionResults[1]["predictedValue"]
    oneStepConfidence = predictionResults[1]["predictionConfidence"]
    fiveStep = predictionResults[5]["predictedValue"]
    fiveStepConfidence = predictionResults[5]["predictionConfidence"]

    result = (oneStep, oneStepConfidence * 100,
              fiveStep, fiveStepConfidence * 100)
    print "1-step: {:16} ({:4.4}%)\t 5-step: {:16} ({:4.4}%)".format(*result)
    results.append(result)

  return results


Instruction:
Can you tell what is the following Python 3 function doing
def _loadDummyModelParameters(self, params):

    for key, value in params.iteritems():
      if type(value) == list:
        index = self.modelIndex % len(params[key])
        self._params[key] = params[key][index]
      else:
        self._params[key] = params[key]

Response: Loads all the parameters for this dummy model.


Instruction:
Make a summary of the following Python 3 code
def _computModelDelay(self):

    # 'delay' and 'sleepModelRange' are mutually exclusive
    if self._params['delay'] is not None \
        and self._params['sleepModelRange'] is not None:
          raise RuntimeError("Only one of 'delay' or "
                             "'sleepModelRange' may be specified")

    # Get the sleepModel range
    if self._sleepModelRange is not None:
      range, delay = self._sleepModelRange.split(':')
      delay = float(delay)
      range = map(int, range.split(','))
      modelIDs = self._jobsDAO.jobGetModelIDs(self._jobID)
      modelIDs.sort()

      range[1] = min(range[1], len(modelIDs))

      # If the model is in range, add the delay
      if self._modelID in modelIDs[range[0]:range[1]]:
        self._delay = delay
    else:
      self._delay = self._params['delay']

Response: Compute the amount of time to delay the run of this model.


Instruction:
Here you have a function in Python 3, explain what it does
def _getMetrics(self):
    metric = None
    if self.metrics is not None:
      metric = self.metrics(self._currentRecordIndex+1)
    elif self.metricValue is not None:
      metric = self.metricValue
    else:
      raise RuntimeError('No metrics or metric value specified for dummy model')

    return {self._optimizeKeyPattern:metric}

Response: Private function that can be overridden by subclasses. It can be overridden by subclasses. It can be overridden by subclasses. It can be overridden by subclasses. It can be overridden by subclasses. It can be overridden by subclasses.


Instruction:
How would you explain what the following Python 3 function does
def run(self):

    self._logger.debug("Starting Dummy Model: modelID=%s;" % (self._modelID))

    # =========================================================================
    # Initialize periodic activities (e.g., for model result updates)
    # =========================================================================
    periodic = self._initPeriodicActivities()

    self._optimizedMetricLabel = self._optimizeKeyPattern
    self._reportMetricLabels = [self._optimizeKeyPattern]

    # =========================================================================
    # Create our top-level loop-control iterator
    # =========================================================================
    if self._iterations >= 0:
      iterTracker = iter(xrange(self._iterations))
    else:
      iterTracker = iter(itertools.count())

    # =========================================================================
    # This gets set in the unit tests. It tells the worker to sys exit
    #  the first N models. This is how we generate orphaned models
    doSysExit = False
    if self._sysExitModelRange is not None:
      modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)
      modelIDs = [x[0] for x in modelAndCounters]
      modelIDs.sort()
      (beg,end) = self._sysExitModelRange
      if self._modelID in modelIDs[int(beg):int(end)]:
        doSysExit = True

    if self._delayModelRange is not None:
      modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)
      modelIDs = [x[0] for x in modelAndCounters]
      modelIDs.sort()
      (beg,end) = self._delayModelRange
      if self._modelID in modelIDs[int(beg):int(end)]:
        time.sleep(10)

      # DEBUG!!!! infinite wait if we have 50 models
      #if len(modelIDs) >= 50:
      #  jobCancel = self._jobsDAO.jobGetFields(self._jobID, ['cancel'])[0]
      #  while not jobCancel:
      #    time.sleep(1)
      #    jobCancel = self._jobsDAO.jobGetFields(self._jobID, ['cancel'])[0]

    if self._errModelRange is not None:
      modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)
      modelIDs = [x[0] for x in modelAndCounters]
      modelIDs.sort()
      (beg,end) = self._errModelRange
      if self._modelID in modelIDs[int(beg):int(end)]:
        raise RuntimeError("Exiting with error due to errModelRange parameter")

    # =========================================================================
    # Delay, if necessary
    if self._delay is not None:
      time.sleep(self._delay)

    # =========================================================================
    # Run it!
    # =========================================================================
    self._currentRecordIndex = 0
    while True:

      # =========================================================================
      # Check if the model should be stopped
      # =========================================================================

      # If killed by a terminator, stop running
      if self._isKilled:
        break

      # If job stops or hypersearch ends, stop running
      if self._isCanceled:
        break

      # If model is mature, stop running ONLY IF  we are not the best model
      # for the job. Otherwise, keep running so we can keep returning
      # predictions to the user
      if self._isMature:
        if not self._isBestModel:
          self._cmpReason = self._jobsDAO.CMPL_REASON_STOPPED
          break
        else:
          self._cmpReason = self._jobsDAO.CMPL_REASON_EOF

      # =========================================================================
      # Get the the next record, and "write it"
      # =========================================================================
      try:
        self._currentRecordIndex = next(iterTracker)
      except StopIteration:
        break

      # "Write" a dummy output value. This is used to test that the batched
      # writing works properly

      self._writePrediction(ModelResult(None, None, None, None))

      periodic.tick()

      # =========================================================================
      # Compute wait times. See if model should exit
      # =========================================================================

      if self.__shouldSysExit(self._currentRecordIndex):
        sys.exit(1)

      # Simulate computation time
      if self._busyWaitTime is not None:
        time.sleep(self._busyWaitTime)
        self.__computeWaitTime()

      # Asked to abort after so many iterations?
      if doSysExit:
        sys.exit(1)

      # Asked to raise a jobFailException?
      if self._jobFailErr:
        raise utils.JobFailException("E10000",
                                      "dummyModel's jobFailErr was True.")

    # =========================================================================
    # Handle final operations
    # =========================================================================
    if self._doFinalize:
      if not self._makeCheckpoint:
        self._model = None

      # Delay finalization operation
      if self._finalDelay is not None:
        time.sleep(self._finalDelay)

      self._finalize()

    self._logger.info("Finished: modelID=%r "% (self._modelID))

    return (self._cmpReason, None)

Response: Runs the OPF task against the given Model instance.


Instruction:
Can you create a Python 3 function that
creates the PredictionLogger object which is an interface to write model results to a permanent storage location.

Response:
def _createPredictionLogger(self):
    """
    Creates the model's PredictionLogger object, which is an interface to write
    model results to a permanent storage location
    """

    class DummyLogger:
      def writeRecord(self, record): pass
      def writeRecords(self, records, progressCB): pass
      def close(self): pass

    self._predictionLogger = DummyLogger()


Instruction:
Create a Python 3 function to
check to see if the model should exit based on the exitAfter dummy parameter

Response:
def __shouldSysExit(self, iteration):
    """
    Checks to see if the model should exit based on the exitAfter dummy
    parameter
    """

    if self._exitAfter is None \
       or iteration < self._exitAfter:
      return False

    results = self._jobsDAO.modelsGetFieldsForJob(self._jobID, ['params'])

    modelIDs = [e[0] for e in results]
    modelNums = [json.loads(e[1][0])['structuredParams']['__model_num'] for e in results]

    sameModelNumbers = filter(lambda x: x[1] == self.modelIndex,
                              zip(modelIDs, modelNums))

    firstModelID = min(zip(*sameModelNumbers)[0])

    return firstModelID == self._modelID


Instruction:
Make a summary of the following Python 3 code
def getDescription(self):

    description = {'name':self.name, 'fields':[f.name for f in self.fields], \
      'numRecords by field':[f.numRecords for f in self.fields]}

    return description

Response: Returns a description of the dataset


Instruction:
Here you have a function in Python 3, explain what it does
def setSeed(self, seed):

    rand.seed(seed)
    np.random.seed(seed)

Response: Sets the random seed and numpy seed


Instruction:
Can you create a Python 3 function that
adds a single field to the dataset.

Response:
def addField(self, name, fieldParams, encoderParams):
    """Add a single field to the dataset.
    Parameters:
    -------------------------------------------------------------------
    name:             The user-specified name of the field
    fieldSpec:        A list of one or more dictionaries specifying parameters
                      to be used for dataClass initialization. Each dict must
                      contain the key 'type' that specifies a distribution for
                      the values in this field
    encoderParams:    Parameters for the field encoder
    """

    assert fieldParams is not None and'type' in fieldParams

    dataClassName = fieldParams.pop('type')
    try:
      dataClass=eval(dataClassName)(fieldParams)

    except TypeError, e:
      print ("#### Error in constructing %s class object. Possibly missing "
              "some required constructor parameters. Parameters "
              "that were provided are: %s" % (dataClass, fieldParams))
      raise

    encoderParams['dataClass']=dataClass
    encoderParams['dataClassName']=dataClassName

    fieldIndex = self.defineField(name, encoderParams)


Instruction:
Can you tell what is the following Python 3 function doing
def addMultipleFields(self, fieldsInfo):
    assert all(x in field for x in ['name', 'fieldSpec', 'encoderParams'] for field \
               in fieldsInfo)

    for spec in fieldsInfo:
      self.addField(spec.pop('name'), spec.pop('fieldSpec'), spec.pop('encoderParams'))

Response: Adds multiple fields to the dataset.


Instruction:
Explain what the following Python 3 code does
def defineField(self, name, encoderParams=None):
    self.fields.append(_field(name, encoderParams))

    return len(self.fields)-1

Response: Define a new field in the encoder.


Instruction:
Here you have a function in Python 3, explain what it does
def setFlag(self, index, flag):
    assert len(self.fields)>index
    self.fields[index].flag=flag

Response: Sets flag for the field at index.


Instruction:
Make a summary of the following Python 3 code
def generateRecord(self, record):
    assert(len(record)==len(self.fields))
    if record is not None:
      for x in range(len(self.fields)):
        self.fields[x].addValue(record[x])

    else:
      for field in self.fields:
        field.addValue(field.dataClass.getNext())

Response: Generates a record. Each value is stored in its respective field.


Instruction:
Write a Python 3 function for
generating multiple records. Refer to definition for generateRecord

Response:
def generateRecords(self, records):
    """Generate multiple records. Refer to definition for generateRecord"""

    if self.verbosity>0: print 'Generating', len(records), 'records...'
    for record in records:
      self.generateRecord(record)


Instruction:
How would you code a function in Python 3 to
return the nth record

Response:
def getRecord(self, n=None):
    """Returns the nth record"""

    if n is None:
      assert len(self.fields)>0
      n = self.fields[0].numRecords-1

    assert (all(field.numRecords>n for field in self.fields))

    record = [field.values[n] for field in self.fields]

    return record


Instruction:
Explain what the following Python 3 code does
def getAllRecords(self):
    values=[]
    numRecords = self.fields[0].numRecords
    assert (all(field.numRecords==numRecords for field in self.fields))

    for x in range(numRecords):
      values.append(self.getRecord(x))

    return values

Response: Returns all the records


Instruction:
Can you write a function in Python 3 where it
encodes a record as a sparse distributed representation.

Response:
def encodeRecord(self, record, toBeAdded=True):
    """Encode a record as a sparse distributed representation
    Parameters:
    --------------------------------------------------------------------
    record:        Record to be encoded
    toBeAdded:     Whether the encodings corresponding to the record are added to
                   the corresponding fields
    """
    encoding=[self.fields[i].encodeValue(record[i], toBeAdded) for i in \
              xrange(len(self.fields))]

    return encoding


Instruction:
Explain what the following Python 3 code does
def encodeAllRecords(self, records=None, toBeAdded=True):
    if records is None:
      records = self.getAllRecords()
    if self.verbosity>0: print 'Encoding', len(records), 'records.'
    encodings = [self.encodeRecord(record, toBeAdded) for record in records]

    return encodings

Response: Encodes a list of records.


Instruction:
Write a Python 3 script for
adding a value to the field i.

Response:
def addValueToField(self, i, value=None):
    """Add 'value' to the field i.
    Parameters:
    --------------------------------------------------------------------
    value:       value to be added
    i:           value is added to field i
    """

    assert(len(self.fields)>i)
    if value is None:
      value = self.fields[i].dataClass.getNext()
      self.fields[i].addValue(value)
      return value

    else: self.fields[i].addValue(value)


Instruction:
Can you implement a function in Python 3 that
adds values to the field i.

Response:
def addValuesToField(self, i, numValues):
    """Add values to the field i."""

    assert(len(self.fields)>i)
    values = [self.addValueToField(i) for n in range(numValues)]
    return values


Instruction:
Can you create a Python 3 function that
returns the sdr for the value at column i

Response:
def getSDRforValue(self, i, j):
    """Returns the sdr for jth value at column i"""
    assert len(self.fields)>i
    assert self.fields[i].numRecords>j
    encoding = self.fields[i].encodings[j]

    return encoding


Instruction:
Implement a function in Python 3 to
return the nth encoding with the predictedField zeroed out

Response:
def getZeroedOutEncoding(self, n):
    """Returns the nth encoding with the predictedField zeroed out"""

    assert all(field.numRecords>n for field in self.fields)

    encoding = np.concatenate([field.encoder.encode(SENTINEL_VALUE_FOR_MISSING_DATA)\
        if field.isPredictedField else field.encodings[n] for field in self.fields])

    return encoding


Instruction:
How would you explain what the following Python 3 function does
def getTotaln(self):

    n = sum([field.n for field in self.fields])
    return n

Response: Returns the cumulative n for all the fields in the dataset


Instruction:
How would you explain what the following Python 3 function does
def getTotalw(self):

    w = sum([field.w for field in self.fields])
    return w

Response: Returns the cumulative w for all the fields in the dataset


Instruction:
Can you write a function in Python 3 where it
returns the nth encoding

Response:
def getEncoding(self, n):
    """Returns the nth encoding"""

    assert (all(field.numEncodings>n for field in self.fields))
    encoding = np.concatenate([field.encodings[n] for field in self.fields])

    return encoding


Instruction:
Write a Python 3 script to
return all the encodings for all the records

Response:
def getAllEncodings(self):
    """Returns encodings for all the records"""

    numEncodings=self.fields[0].numEncodings
    assert (all(field.numEncodings==numEncodings for field in self.fields))
    encodings = [self.getEncoding(index) for index in range(numEncodings)]

    return encodings


Instruction:
Here you have a function in Python 3, explain what it does
def saveRecords(self, path='myOutput'):
    numRecords = self.fields[0].numRecords
    assert (all(field.numRecords==numRecords for field in self.fields))

    import csv
    with open(path+'.csv', 'wb') as f:
      writer = csv.writer(f)
      writer.writerow(self.getAllFieldNames())
      writer.writerow(self.getAllDataTypes())
      writer.writerow(self.getAllFlags())
      writer.writerows(self.getAllRecords())
    if self.verbosity>0:
      print '******', numRecords,'records exported in numenta format to file:',\
                path,'******\n'

Response: Export all the records into a csv file in numenta format.


Instruction:
Make a summary of the following Python 3 code
def removeAllRecords(self):

    for field in self.fields:
      field.encodings, field.values=[], []
      field.numRecords, field.numEncodings= (0, 0)

Response: Deletes all the records in the dataset


Instruction:
Can you generate the documentation for the following Python 3 function
def encodeValue(self, value, toBeAdded=True):

    encodedValue = np.array(self.encoder.encode(value), dtype=realDType)

    if toBeAdded:
      self.encodings.append(encodedValue)
      self.numEncodings+=1

    return encodedValue

Response: Encode a value using the encoder parameters of the Field


Instruction:
Write a Python 3 script for
setting up the dataTypes and initialize encoders

Response:
def _setTypes(self, encoderSpec):
    """Set up the dataTypes and initialize encoders"""

    if self.encoderType is None:
      if self.dataType in ['int','float']:
        self.encoderType='adaptiveScalar'
      elif self.dataType=='string':
        self.encoderType='category'
      elif self.dataType in ['date', 'datetime']:
        self.encoderType='date'

    if self.dataType is None:
      if self.encoderType in ['scalar','adaptiveScalar']:
        self.dataType='float'
      elif self.encoderType in ['category', 'enumeration']:
        self.dataType='string'
      elif self.encoderType in ['date', 'datetime']:
        self.dataType='datetime'


Instruction:
Can you generate the documentation for the following Python 3 function
def _initializeEncoders(self, encoderSpec):

    #Initializing scalar encoder
    if self.encoderType in ['adaptiveScalar', 'scalar']:
      if 'minval' in encoderSpec:
        self.minval = encoderSpec.pop('minval')
      else: self.minval=None
      if 'maxval' in encoderSpec:
        self.maxval = encoderSpec.pop('maxval')
      else: self.maxval = None
      self.encoder=adaptive_scalar.AdaptiveScalarEncoder(name='AdaptiveScalarEncoder', \
                                                         w=self.w, n=self.n, minval=self.minval, maxval=self.maxval, periodic=False, forced=True)

    #Initializing category encoder
    elif self.encoderType=='category':
      self.encoder=sdr_category.SDRCategoryEncoder(name='categoryEncoder', \
                                                   w=self.w, n=self.n)

    #Initializing date encoder
    elif self.encoderType in ['date', 'datetime']:
      self.encoder=date.DateEncoder(name='dateEncoder')
    else:
      raise RuntimeError('Error in constructing class object. Either encoder type'
          'or dataType must be specified')

Response: Initializes the encoders for the current encoder object based on the encoderSpec.


Instruction:
Implement a Python 3 function for
returning a numpy array of scalar values for the given input.

Response:
def getScalars(self, input):
    """ See method description in base.py """
    if input == SENTINEL_VALUE_FOR_MISSING_DATA:
      return numpy.array([None])
    else:
      return numpy.array([self.categoryToIndex.get(input, 0)])


Instruction:
Given the following Python 3 function, write the documentation
def getBucketIndices(self, input):

    # Get the bucket index from the underlying scalar encoder
    if input == SENTINEL_VALUE_FOR_MISSING_DATA:
      return [None]
    else:
      return self.encoder.getBucketIndices(self.categoryToIndex.get(input, 0))

Response: Get the bucket indices for the given input.


Instruction:
Can you create a Python 3 function that
sees the function description in base.py

Response:
def decode(self, encoded, parentFieldName=''):
    """ See the function description in base.py
    """

    # Get the scalar values from the underlying scalar encoder
    (fieldsDict, fieldNames) = self.encoder.decode(encoded)
    if len(fieldsDict) == 0:
      return (fieldsDict, fieldNames)

    # Expect only 1 field
    assert(len(fieldsDict) == 1)

    # Get the list of categories the scalar values correspond to and
    #  generate the description from the category name(s).
    (inRanges, inDesc) = fieldsDict.values()[0]
    outRanges = []
    desc = ""
    for (minV, maxV) in inRanges:
      minV = int(round(minV))
      maxV = int(round(maxV))
      outRanges.append((minV, maxV))
      while minV <= maxV:
        if len(desc) > 0:
          desc += ", "
        desc += self.indexToCategory[minV]
        minV += 1

    # Return result
    if parentFieldName != '':
      fieldName = "%s.%s" % (parentFieldName, self.name)
    else:
      fieldName = self.name
    return ({fieldName: (outRanges, desc)}, [fieldName])


Instruction:
Can you generate the documentation for the following Python 3 function
def closenessScores(self, expValues, actValues, fractional=True,):

    expValue = expValues[0]
    actValue = actValues[0]

    if expValue == actValue:
      closeness = 1.0
    else:
      closeness = 0.0

    if not fractional:
      closeness = 1.0 - closeness

    return numpy.array([closeness])

Response: Returns the closeness scores of the given expValues and actValues.


Instruction:
Write a Python 3 function that can
return a list of all bucket values in the encoder.

Response:
def getBucketValues(self):
    """ See the function description in base.py """

    if self._bucketValues is None:
      numBuckets = len(self.encoder.getBucketValues())
      self._bucketValues = []
      for bucketIndex in range(numBuckets):
        self._bucketValues.append(self.getBucketInfo([bucketIndex])[0].value)

    return self._bucketValues


Instruction:
Can you create a Python 3 function that
returns encoderResult for the given buckets.

Response:
def getBucketInfo(self, buckets):
    """ See the function description in base.py
    """

    # For the category encoder, the bucket index is the category index
    bucketInfo = self.encoder.getBucketInfo(buckets)[0]

    categoryIndex = int(round(bucketInfo.value))
    category = self.indexToCategory[categoryIndex]

    return [EncoderResult(value=category, scalar=categoryIndex,
                         encoding=bucketInfo.encoding)]


Instruction:
Here you have a function in Python 3, explain what it does
def topDownCompute(self, encoded):

    encoderResult = self.encoder.topDownCompute(encoded)[0]
    value = encoderResult.value
    categoryIndex = int(round(value))
    category = self.indexToCategory[categoryIndex]

    return EncoderResult(value=category, scalar=categoryIndex,
                         encoding=encoderResult.encoding)

Response: Compute the encoder for the given encoded string.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def loadExperiment(path):
  if not os.path.isdir(path):
    path = os.path.dirname(path)
  descriptionPyModule = loadExperimentDescriptionScriptFromDir(path)
  expIface = getExperimentDescriptionInterfaceFromModule(descriptionPyModule)
  return expIface.getModelDescription(), expIface.getModelControl()

Response: Loads the experiment description file from the path.


Instruction:
Implement a function in Python 3 to
load the experiment description python script from the given experiment directory.

Response:
def loadExperimentDescriptionScriptFromDir(experimentDir):
  """ Loads the experiment description python script from the given experiment
  directory.

  :param experimentDir: (string) experiment directory path

  :returns:        module of the loaded experiment description scripts
  """
  descriptionScriptPath = os.path.join(experimentDir, "description.py")
  module = _loadDescriptionFile(descriptionScriptPath)
  return module


Instruction:
Here you have a function in Python 3, explain what it does
def getExperimentDescriptionInterfaceFromModule(module):
  result = module.descriptionInterface
  assert isinstance(result, exp_description_api.DescriptionIface), \
         "expected DescriptionIface-based instance, but got %s" % type(result)

  return result

Response: Returns the description interface from the given module.


Instruction:
Create a Python 3 function to
load a description. py file and returns it as a module.

Response:
def _loadDescriptionFile(descriptionPyPath):
  """Loads a description file and returns it as a module.

  descriptionPyPath: path of description.py file to load
  """
  global g_descriptionImportCount

  if not os.path.isfile(descriptionPyPath):
    raise RuntimeError(("Experiment description file %s does not exist or " + \
                        "is not a file") % (descriptionPyPath,))

  mod = imp.load_source("pf_description%d" % g_descriptionImportCount,
                        descriptionPyPath)
  g_descriptionImportCount += 1

  if not hasattr(mod, "descriptionInterface"):
    raise RuntimeError("Experiment description file %s does not define %s" % \
                       (descriptionPyPath, "descriptionInterface"))

  if not isinstance(mod.descriptionInterface, exp_description_api.DescriptionIface):
    raise RuntimeError(("Experiment description file %s defines %s but it " + \
                        "is not DescriptionIface-based") % \
                            (descriptionPyPath, name))

  return mod


Instruction:
Can you generate the documentation for the following Python 3 function
def update(self, modelID, modelParams, modelParamsHash, metricResult,
             completed, completionReason, matured, numRecords):
    """ Insert a new entry or update an existing one. If this is an update
    of an existing entry, then modelParams will be None

    Parameters:
    --------------------------------------------------------------------
    modelID:       globally unique modelID of this model
    modelParams:    params dict for this model, or None if this is just an update
                    of a model that it already previously reported on.

                    See the comments for the createModels() method for
                    a description of this dict.

    modelParamsHash:  hash of the modelParams dict, generated by the worker
                    that put it into the model database.
    metricResult:   value on the optimizeMetric for this model.
                    May be None if we have no results yet.
    completed:      True if the model has completed evaluation, False if it
                      is still running (and these are online results)
    completionReason: One of the ClientJobsDAO.CMPL_REASON_XXX equates
    matured:        True if this model has matured
    numRecords:     Number of records that have been processed so far by this
                      model.

    retval: Canonicalized result on the optimize metric
    """
    # The modelParamsHash must always be provided - it can change after a
    #  model is inserted into the models table if it got detected as an
    #  orphan
    assert (modelParamsHash is not None)

    # We consider a model metricResult as "final" if it has completed or
    #  matured. By default, assume anything that has completed has matured
    if completed:
      matured = True

    # Get the canonicalized optimize metric results. For this metric, lower
    #  is always better
    if metricResult is not None and matured and \
                       completionReason in [ClientJobsDAO.CMPL_REASON_EOF,
                                            ClientJobsDAO.CMPL_REASON_STOPPED]:
      # Canonicalize the error score so that lower is better
      if self._hsObj._maximize:
        errScore = -1 * metricResult
      else:
        errScore = metricResult

      if errScore < self._bestResult:
        self._bestResult = errScore
        self._bestModelID = modelID
        self._hsObj.logger.info("New best model after %d evaluations: errScore "
              "%g on model %s" % (len(self._allResults), self._bestResult,
                                  self._bestModelID))

    else:
      errScore = numpy.inf

    # If this model completed with an unacceptable completion reason, set the
    #  errScore to infinite and essentially make this model invisible to
    #  further queries
    if completed and completionReason in [ClientJobsDAO.CMPL_REASON_ORPHAN]:
      errScore = numpy.inf
      hidden = True
    else:
      hidden = False

    # Update our set of erred models and completed models. These are used
    #  to determine if we should abort the search because of too many errors
    if completed:
      self._completedModels.add(modelID)
      self._numCompletedModels = len(self._completedModels)
      if completionReason == ClientJobsDAO.CMPL_REASON_ERROR:
        self._errModels.add(modelID)
        self._numErrModels = len(self._errModels)

    # Are we creating a new entry?
    wasHidden = False
    if modelID not in self._modelIDToIdx:
      assert (modelParams is not None)
      entry = dict(modelID=modelID, modelParams=modelParams,
                   modelParamsHash=modelParamsHash,
                   errScore=errScore, completed=completed,
                   matured=matured, numRecords=numRecords, hidden=hidden)
      self._allResults.append(entry)
      entryIdx = len(self._allResults) - 1
      self._modelIDToIdx[modelID] = entryIdx

      self._paramsHashToIndexes[modelParamsHash] = entryIdx

      swarmId = modelParams['particleState']['swarmId']
      if not hidden:
        # Update the list of particles in each swarm
        if swarmId in self._swarmIdToIndexes:
          self._swarmIdToIndexes[swarmId].append(entryIdx)
        else:
          self._swarmIdToIndexes[swarmId] = [entryIdx]

        # Update number of particles at each generation in this swarm
        genIdx = modelParams['particleState']['genIdx']
        numPsEntry = self._swarmNumParticlesPerGeneration.get(swarmId, [0])
        while genIdx >= len(numPsEntry):
          numPsEntry.append(0)
        numPsEntry[genIdx] += 1
        self._swarmNumParticlesPerGeneration[swarmId] = numPsEntry

    # Replacing an existing one
    else:
      entryIdx = self._modelIDToIdx.get(modelID, None)
      assert (entryIdx is not None)
      entry = self._allResults[entryIdx]
      wasHidden = entry['hidden']

      # If the paramsHash changed, note that. This can happen for orphaned
      #  models
      if entry['modelParamsHash'] != modelParamsHash:

        self._paramsHashToIndexes.pop(entry['modelParamsHash'])
        self._paramsHashToIndexes[modelParamsHash] = entryIdx
        entry['modelParamsHash'] = modelParamsHash

      # Get the model params, swarmId, and genIdx
      modelParams = entry['modelParams']
      swarmId = modelParams['particleState']['swarmId']
      genIdx = modelParams['particleState']['genIdx']

      # If this particle just became hidden, remove it from our swarm counts
      if hidden and not wasHidden:
        assert (entryIdx in self._swarmIdToIndexes[swarmId])
        self._swarmIdToIndexes[swarmId].remove(entryIdx)
        self._swarmNumParticlesPerGeneration[swarmId][genIdx] -= 1

      # Update the entry for the latest info
      entry['errScore']  = errScore
      entry['completed'] = completed
      entry['matured'] = matured
      entry['numRecords'] = numRecords
      entry['hidden'] = hidden

    # Update the particle best errScore
    particleId = modelParams['particleState']['id']
    genIdx = modelParams['particleState']['genIdx']
    if matured and not hidden:
      (oldResult, pos) = self._particleBest.get(particleId, (numpy.inf, None))
      if errScore < oldResult:
        pos = Particle.getPositionFromState(modelParams['particleState'])
        self._particleBest[particleId] = (errScore, pos)

    # Update the particle latest generation index
    prevGenIdx = self._particleLatestGenIdx.get(particleId, -1)
    if not hidden and genIdx > prevGenIdx:
      self._particleLatestGenIdx[particleId] = genIdx
    elif hidden and not wasHidden and genIdx == prevGenIdx:
      self._particleLatestGenIdx[particleId] = genIdx-1

    # Update the swarm best score
    if not hidden:
      swarmId = modelParams['particleState']['swarmId']
      if not swarmId in self._swarmBestOverall:
        self._swarmBestOverall[swarmId] = []

      bestScores = self._swarmBestOverall[swarmId]
      while genIdx >= len(bestScores):
        bestScores.append((None, numpy.inf))
      if errScore < bestScores[genIdx][1]:
        bestScores[genIdx] = (modelID, errScore)

    # Update the self._modifiedSwarmGens flags to support the
    #   getMaturedSwarmGenerations() call.
    if not hidden:
      key = (swarmId, genIdx)
      if not key in self._maturedSwarmGens:
        self._modifiedSwarmGens.add(key)

    return errScore

Response: Insert a new entry or update an existing entry.


Instruction:
Explain what the following Python 3 code does
def getModelIDFromParamsHash(self, paramsHash):
    entryIdx = self. _paramsHashToIndexes.get(paramsHash, None)
    if entryIdx is not None:
      return self._allResults[entryIdx]['modelID']
    else:
      return None

Response: Returns the modelID of the model with the given paramsHash or None if not found.


Instruction:
Make a summary of the following Python 3 code
def numModels(self, swarmId=None, includeHidden=False):
    # Count all models
    if includeHidden:
      if swarmId is None:
        return len(self._allResults)

      else:
        return len(self._swarmIdToIndexes.get(swarmId, []))
    # Only count non-hidden models
    else:
      if swarmId is None:
        entries = self._allResults
      else:
        entries = [self._allResults[entryIdx]
                   for entryIdx in self._swarmIdToIndexes.get(swarmId,[])]

      return len([entry for entry in entries if not entry['hidden']])

Response: Return the total number of models in a specific swarm.


Instruction:
Can you create a Python 3 function that
returns the model ID of the model with the best result so far and the best score on the optimize metric.

Response:
def bestModelIdAndErrScore(self, swarmId=None, genIdx=None):
    """Return the model ID of the model with the best result so far and
    it's score on the optimize metric. If swarm is None, then it returns
    the global best, otherwise it returns the best for the given swarm
    for all generatons up to and including genIdx.

    Parameters:
    ---------------------------------------------------------------------
    swarmId:  A string representation of the sorted list of encoders in this
                 swarm. For example '__address_encoder.__gym_encoder'
    genIdx:   consider the best in all generations up to and including this
                generation if not None.
    retval:  (modelID, result)
    """
    if swarmId is None:
      return (self._bestModelID, self._bestResult)

    else:
      if swarmId not in self._swarmBestOverall:
        return (None, numpy.inf)

      # Get the best score, considering the appropriate generations
      genScores = self._swarmBestOverall[swarmId]
      bestModelId = None
      bestScore = numpy.inf

      for (i, (modelId, errScore)) in enumerate(genScores):
        if genIdx is not None and i > genIdx:
          break
        if errScore < bestScore:
          bestScore = errScore
          bestModelId = modelId

      return (bestModelId, bestScore)


Instruction:
Write a Python 3 function that can
return particle info for a specific modelId.

Response:
def getParticleInfo(self, modelId):
    """Return particle info for a specific modelId.

    Parameters:
    ---------------------------------------------------------------------
    modelId:  which model Id

    retval:  (particleState, modelId, errScore, completed, matured)
    """
    entry = self._allResults[self._modelIDToIdx[modelId]]
    return (entry['modelParams']['particleState'], modelId, entry['errScore'],
            entry['completed'], entry['matured'])


Instruction:
How would you code a function in Python 3 to
return a list of particleStates modelIds errScores completed and matured states for all the models in this swarm.

Response:
def getParticleInfos(self, swarmId=None, genIdx=None, completed=None,
                       matured=None, lastDescendent=False):
    """Return a list of particleStates for all particles we know about in
    the given swarm, their model Ids, and metric results.

    Parameters:
    ---------------------------------------------------------------------
    swarmId:  A string representation of the sorted list of encoders in this
                 swarm. For example '__address_encoder.__gym_encoder'

    genIdx:  If not None, only return particles at this specific generation
                  index.

    completed:   If not None, only return particles of the given state (either
                completed if 'completed' is True, or running if 'completed'
                is false

    matured:   If not None, only return particles of the given state (either
                matured if 'matured' is True, or not matured if 'matured'
                is false. Note that any model which has completed is also
                considered matured.

    lastDescendent: If True, only return particles that are the last descendent,
                that is, the highest generation index for a given particle Id

    retval:  (particleStates, modelIds, errScores, completed, matured)
              particleStates: list of particleStates
              modelIds: list of modelIds
              errScores: list of errScores, numpy.inf is plugged in
                              if we don't have a result yet
              completed: list of completed booleans
              matured: list of matured booleans
    """
    # The indexes of all the models in this swarm. This list excludes hidden
    #  (orphaned) models.
    if swarmId is not None:
      entryIdxs = self._swarmIdToIndexes.get(swarmId, [])
    else:
      entryIdxs = range(len(self._allResults))
    if len(entryIdxs) == 0:
      return ([], [], [], [], [])

    # Get the particles of interest
    particleStates = []
    modelIds = []
    errScores = []
    completedFlags = []
    maturedFlags = []
    for idx in entryIdxs:
      entry = self._allResults[idx]

      # If this entry is hidden (i.e. it was an orphaned model), it should
      #  not be in this list
      if swarmId is not None:
        assert (not entry['hidden'])

      # Get info on this model
      modelParams = entry['modelParams']
      isCompleted = entry['completed']
      isMatured = entry['matured']
      particleState = modelParams['particleState']
      particleGenIdx = particleState['genIdx']
      particleId = particleState['id']

      if genIdx is not None and particleGenIdx != genIdx:
        continue

      if completed is not None and (completed != isCompleted):
        continue

      if matured is not None and (matured != isMatured):
        continue

      if lastDescendent \
              and (self._particleLatestGenIdx[particleId] != particleGenIdx):
        continue

      # Incorporate into return values
      particleStates.append(particleState)
      modelIds.append(entry['modelID'])
      errScores.append(entry['errScore'])
      completedFlags.append(isCompleted)
      maturedFlags.append(isMatured)

    return (particleStates, modelIds, errScores, completedFlags, maturedFlags)


Instruction:
Explain what the following Python 3 code does
def getOrphanParticleInfos(self, swarmId, genIdx):

    entryIdxs = range(len(self._allResults))
    if len(entryIdxs) == 0:
      return ([], [], [], [], [])

    # Get the particles of interest
    particleStates = []
    modelIds = []
    errScores = []
    completedFlags = []
    maturedFlags = []
    for idx in entryIdxs:

      # Get info on this model
      entry = self._allResults[idx]
      if not entry['hidden']:
        continue

      modelParams = entry['modelParams']
      if modelParams['particleState']['swarmId'] != swarmId:
        continue

      isCompleted = entry['completed']
      isMatured = entry['matured']
      particleState = modelParams['particleState']
      particleGenIdx = particleState['genIdx']
      particleId = particleState['id']

      if genIdx is not None and particleGenIdx != genIdx:
        continue

      # Incorporate into return values
      particleStates.append(particleState)
      modelIds.append(entry['modelID'])
      errScores.append(entry['errScore'])
      completedFlags.append(isCompleted)
      maturedFlags.append(isMatured)

    return (particleStates, modelIds, errScores, completedFlags, maturedFlags)

Response: Returns a list of particleStates modelIds errScores completed flags and matured flags for all particles in the given swarm generation.


Instruction:
Can you create a Python 3 function that
returns the index of the first non - full generation in the given swarm.

Response:
def firstNonFullGeneration(self, swarmId, minNumParticles):
    """ Return the generation index of the first generation in the given
    swarm that does not have numParticles particles in it, either still in the
    running state or completed. This does not include orphaned particles.

    Parameters:
    ---------------------------------------------------------------------
    swarmId:  A string representation of the sorted list of encoders in this
                 swarm. For example '__address_encoder.__gym_encoder'
    minNumParticles: minium number of partices required for a full
                  generation.

    retval:  generation index, or None if no particles at all.
    """

    if not swarmId in self._swarmNumParticlesPerGeneration:
      return None

    numPsPerGen = self._swarmNumParticlesPerGeneration[swarmId]

    numPsPerGen = numpy.array(numPsPerGen)
    firstNonFull = numpy.where(numPsPerGen < minNumParticles)[0]
    if len(firstNonFull) == 0:
      return len(numPsPerGen)
    else:
      return firstNonFull[0]


Instruction:
Can you write a function in Python 3 where it
returns a dict of the errors obtained on models that were run with a PermuteChoice variable.

Response:
def getResultsPerChoice(self, swarmId, maxGenIdx, varName):
    """ Return a dict of the errors obtained on models that were run with
    each value from a PermuteChoice variable.

    For example, if a PermuteChoice variable has the following choices:
      ['a', 'b', 'c']

    The dict will have 3 elements. The keys are the stringified choiceVars,
    and each value is tuple containing (choiceVar, errors) where choiceVar is
    the original form of the choiceVar (before stringification) and errors is
    the list of errors received from models that used the specific choice:
    retval:
      ['a':('a', [0.1, 0.2, 0.3]), 'b':('b', [0.5, 0.1, 0.6]), 'c':('c', [])]

    Parameters:
    ---------------------------------------------------------------------
    swarmId:  swarm Id of the swarm to retrieve info from
    maxGenIdx: max generation index to consider from other models, ignored
                if None
    varName:  which variable to retrieve

    retval:  list of the errors obtained from each choice.
    """
    results = dict()
    # Get all the completed particles in this swarm
    (allParticles, _, resultErrs, _, _) = self.getParticleInfos(swarmId,
                                              genIdx=None, matured=True)

    for particleState, resultErr in itertools.izip(allParticles, resultErrs):
      # Consider this generation?
      if maxGenIdx is not None:
        if particleState['genIdx'] > maxGenIdx:
          continue

      # Ignore unless this model completed successfully
      if resultErr == numpy.inf:
        continue

      position = Particle.getPositionFromState(particleState)
      varPosition = position[varName]
      varPositionStr = str(varPosition)
      if varPositionStr in results:
        results[varPositionStr][1].append(resultErr)
      else:
        results[varPositionStr] = (varPosition, [resultErr])

    return results


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _getStreamDef(self, modelDescription):
    #--------------------------------------------------------------------------
    # Generate the string containing the aggregation settings.
    aggregationPeriod = {
        'days': 0,
        'hours': 0,
        'microseconds': 0,
        'milliseconds': 0,
        'minutes': 0,
        'months': 0,
        'seconds': 0,
        'weeks': 0,
        'years': 0,
    }

    # Honor any overrides provided in the stream definition
    aggFunctionsDict = {}
    if 'aggregation' in modelDescription['streamDef']:
      for key in aggregationPeriod.keys():
        if key in modelDescription['streamDef']['aggregation']:
          aggregationPeriod[key] = modelDescription['streamDef']['aggregation'][key]
      if 'fields' in modelDescription['streamDef']['aggregation']:
        for (fieldName, func) in modelDescription['streamDef']['aggregation']['fields']:
          aggFunctionsDict[fieldName] = str(func)

    # Do we have any aggregation at all?
    hasAggregation = False
    for v in aggregationPeriod.values():
      if v != 0:
        hasAggregation = True
        break

    # Convert the aggFunctionsDict to a list
    aggFunctionList = aggFunctionsDict.items()
    aggregationInfo = dict(aggregationPeriod)
    aggregationInfo['fields'] = aggFunctionList

    streamDef = copy.deepcopy(modelDescription['streamDef'])
    streamDef['aggregation'] = copy.deepcopy(aggregationInfo)
    return streamDef

Response: Generate the stream definition based on the model description


Instruction:
Make a summary of the following Python 3 code
def close(self):
    if self._tempDir is not None and os.path.isdir(self._tempDir):
      self.logger.debug("Removing temporary directory %r", self._tempDir)
      shutil.rmtree(self._tempDir)
      self._tempDir = None

    return

Response: Deletes temporary system objects and files.


Instruction:
Here you have a function in Python 3, explain what it does
def _readPermutationsFile(self, filename, modelDescription):
    # Open and execute the permutations file
    vars = {}

    permFile = execfile(filename, globals(), vars)

    # Read in misc info.
    self._reportKeys = vars.get('report', [])
    self._filterFunc = vars.get('permutationFilter', None)
    self._dummyModelParamsFunc = vars.get('dummyModelParams', None)
    self._predictedField = None   # default
    self._predictedFieldEncoder = None   # default
    self._fixedFields = None # default

    # The fastSwarm variable, if present, contains the params from a best
    #  model from a previous swarm. If present, use info from that to seed
    #  a fast swarm
    self._fastSwarmModelParams = vars.get('fastSwarmModelParams', None)
    if self._fastSwarmModelParams is not None:
      encoders = self._fastSwarmModelParams['structuredParams']['modelParams']\
                  ['sensorParams']['encoders']
      self._fixedFields = []
      for fieldName in encoders:
        if encoders[fieldName] is not None:
          self._fixedFields.append(fieldName)

    if 'fixedFields' in vars:
      self._fixedFields = vars['fixedFields']

    # Get min number of particles per swarm from either permutations file or
    # config.
    self._minParticlesPerSwarm = vars.get('minParticlesPerSwarm')
    if self._minParticlesPerSwarm  == None:
      self._minParticlesPerSwarm = Configuration.get(
                                      'nupic.hypersearch.minParticlesPerSwarm')
    self._minParticlesPerSwarm = int(self._minParticlesPerSwarm)

    # Enable logic to kill off speculative swarms when an earlier sprint
    #  has found that it contains poorly performing field combination?
    self._killUselessSwarms = vars.get('killUselessSwarms', True)

    # The caller can request that the predicted field ALWAYS be included ("yes")
    #  or optionally include ("auto"). The setting of "no" is N/A and ignored
    #  because in that case the encoder for the predicted field will not even
    #  be present in the permutations file.
    # When set to "yes", this will force the first sprint to try the predicted
    #  field only (the legacy mode of swarming).
    # When set to "auto", the first sprint tries all possible fields (one at a
    #  time) in the first sprint.
    self._inputPredictedField = vars.get("inputPredictedField", "yes")

    # Try all possible 3-field combinations? Normally, we start with the best
    #  2-field combination as a base. When this flag is set though, we try
    #  all possible 3-field combinations which takes longer but can find a
    #  better model.
    self._tryAll3FieldCombinations = vars.get('tryAll3FieldCombinations', False)

    # Always include timestamp fields in the 3-field swarms?
    # This is a less compute intensive version of tryAll3FieldCombinations.
    # Instead of trying ALL possible 3 field combinations, it just insures
    # that the timestamp fields (dayOfWeek, timeOfDay, weekend) are never left
    # out when generating the 3-field swarms.
    self._tryAll3FieldCombinationsWTimestamps = vars.get(
                                'tryAll3FieldCombinationsWTimestamps', False)

    # Allow the permutations file to override minFieldContribution. This would
    #  be set to a negative number for large swarms so that you don't disqualify
    #  a field in an early sprint just because it did poorly there. Sometimes,
    #  a field that did poorly in an early sprint could help accuracy when
    #  added in a later sprint
    minFieldContribution = vars.get('minFieldContribution', None)
    if minFieldContribution is not None:
      self._minFieldContribution = minFieldContribution

    # Allow the permutations file to override maxBranching.
    maxBranching = vars.get('maxFieldBranching', None)
    if maxBranching is not None:
      self._maxBranching = maxBranching

    # Read in the optimization info.
    if 'maximize' in vars:
      self._optimizeKey = vars['maximize']
      self._maximize = True
    elif 'minimize' in vars:
      self._optimizeKey = vars['minimize']
      self._maximize = False
    else:
      raise RuntimeError("Permutations file '%s' does not include a maximize"
                         " or minimize metric.")

    # The permutations file is the new location for maxModels. The old location,
    #  in the jobParams is deprecated.
    maxModels = vars.get('maxModels')
    if maxModels is not None:
      if self._maxModels is None:
        self._maxModels = maxModels
      else:
        raise RuntimeError('It is an error to specify maxModels both in the job'
                ' params AND in the permutations file.')

    # Figure out if what kind of search this is:
    #
    #  If it's a temporal prediction search:
    #    the first sprint has 1 swarm, with just the predicted field
    #  elif it's a spatial prediction search:
    #    the first sprint has N swarms, each with predicted field + one
    #    other field.
    #  elif it's a classification search:
    #    the first sprint has N swarms, each with 1 field
    inferenceType = modelDescription['modelParams']['inferenceType']
    if not InferenceType.validate(inferenceType):
      raise ValueError("Invalid inference type %s" %inferenceType)

    if inferenceType in [InferenceType.TemporalMultiStep,
                         InferenceType.NontemporalMultiStep]:
      # If it does not have a separate encoder for the predicted field that
      #  goes to the classifier, it is a legacy multi-step network
      classifierOnlyEncoder = None
      for encoder in modelDescription["modelParams"]["sensorParams"]\
                    ["encoders"].values():
        if encoder.get("classifierOnly", False) \
             and encoder["fieldname"] == vars.get('predictedField', None):
          classifierOnlyEncoder = encoder
          break

      if classifierOnlyEncoder is None or self._inputPredictedField=="yes":
        # If we don't have a separate encoder for the classifier (legacy
        #  MultiStep) or the caller explicitly wants to include the predicted
        #  field, then use the legacy temporal search methodology.
        self._searchType = HsSearchType.legacyTemporal
      else:
        self._searchType = HsSearchType.temporal

    elif inferenceType in [InferenceType.TemporalNextStep,
                         InferenceType.TemporalAnomaly]:
      self._searchType = HsSearchType.legacyTemporal

    elif inferenceType in (InferenceType.TemporalClassification,
                            InferenceType.NontemporalClassification):
      self._searchType = HsSearchType.classification

    else:
      raise RuntimeError("Unsupported inference type: %s" % inferenceType)

    # Get the predicted field. Note that even classification experiments
    #  have a "predicted" field - which is the field that contains the
    #  classification value.
    self._predictedField = vars.get('predictedField', None)
    if self._predictedField is None:
      raise RuntimeError("Permutations file '%s' does not have the required"
                         " 'predictedField' variable" % filename)

    # Read in and validate the permutations dict
    if 'permutations' not in vars:
      raise RuntimeError("Permutations file '%s' does not define permutations" % filename)

    if not isinstance(vars['permutations'], dict):
      raise RuntimeError("Permutations file '%s' defines a permutations variable "
                         "but it is not a dict")

    self._encoderNames = []
    self._permutations = vars['permutations']
    self._flattenedPermutations = dict()
    def _flattenPermutations(value, keys):
      if ':' in keys[-1]:
        raise RuntimeError("The permutation variable '%s' contains a ':' "
                           "character, which is not allowed.")
      flatKey = _flattenKeys(keys)
      if isinstance(value, PermuteEncoder):
        self._encoderNames.append(flatKey)

        # If this is the encoder for the predicted field, save its name.
        if value.fieldName == self._predictedField:
          self._predictedFieldEncoder = flatKey

        # Store the flattened representations of the variables within the
        # encoder.
        for encKey, encValue in value.kwArgs.iteritems():
          if isinstance(encValue, PermuteVariable):
            self._flattenedPermutations['%s:%s' % (flatKey, encKey)] = encValue
      elif isinstance(value, PermuteVariable):
        self._flattenedPermutations[flatKey] = value

      else:
        if isinstance(value, PermuteVariable):
          self._flattenedPermutations[key] = value
    rApply(self._permutations, _flattenPermutations)

Response: Reads the permutations file and initializes the internal state of the object.


Instruction:
Can you write a function in Python 3 where it
checks if any orphaned models are in the results DB and if so updates them accordingly.

Response:
def _checkForOrphanedModels (self):
    """If there are any models that haven't been updated in a while, consider
    them dead, and mark them as hidden in our resultsDB. We also change the
    paramsHash and particleHash of orphaned models so that we can
    re-generate that particle and/or model again if we desire.

    Parameters:
    ----------------------------------------------------------------------
    retval:

    """

    self.logger.debug("Checking for orphaned models older than %s" % \
                     (self._modelOrphanIntervalSecs))

    while True:
      orphanedModelId = self._cjDAO.modelAdoptNextOrphan(self._jobID,
                                                self._modelOrphanIntervalSecs)
      if orphanedModelId is None:
        return

      self.logger.info("Removing orphaned model: %d" % (orphanedModelId))

      # Change the model hash and params hash as stored in the models table so
      #  that we can insert a new model with the same paramsHash
      for attempt in range(100):
        paramsHash = hashlib.md5("OrphanParams.%d.%d" % (orphanedModelId,
                                                         attempt)).digest()
        particleHash = hashlib.md5("OrphanParticle.%d.%d" % (orphanedModelId,
                                                          attempt)).digest()
        try:
          self._cjDAO.modelSetFields(orphanedModelId,
                                   dict(engParamsHash=paramsHash,
                                        engParticleHash=particleHash))
          success = True
        except:
          success = False
        if success:
          break
      if not success:
        raise RuntimeError("Unexpected failure to change paramsHash and "
                           "particleHash of orphaned model")

      # Mark this model as complete, with reason "orphaned"
      self._cjDAO.modelSetCompleted(modelID=orphanedModelId,
                    completionReason=ClientJobsDAO.CMPL_REASON_ORPHAN,
                    completionMsg="Orphaned")

      # Update our results DB immediately, rather than wait for the worker
      #  to inform us. This insures that the getParticleInfos() calls we make
      #  below don't include this particle. Setting the metricResult to None
      #  sets it to worst case
      self._resultsDB.update(modelID=orphanedModelId,
                             modelParams=None,
                             modelParamsHash=paramsHash,
                             metricResult=None,
                             completed = True,
                             completionReason = ClientJobsDAO.CMPL_REASON_ORPHAN,
                             matured = True,
                             numRecords = 0)


Instruction:
Can you implement a function in Python 3 that
updates the state of the hypersearch with the current state of the current set of hypersearch swarms.

Response:
def _hsStatePeriodicUpdate(self, exhaustedSwarmId=None):
    """
    Periodically, check to see if we should remove a certain field combination
    from evaluation (because it is doing so poorly) or move on to the next
    sprint (add in more fields).

    This method is called from _getCandidateParticleAndSwarm(), which is called
    right before we try and create a new model to run.

    Parameters:
    -----------------------------------------------------------------------
    removeSwarmId:     If not None, force a change to the current set of active
                      swarms by removing this swarm. This is used in situations
                      where we can't find any new unique models to create in
                      this swarm. In these situations, we update the hypersearch
                      state regardless of the timestamp of the last time another
                      worker updated it.

    """
    if self._hsState is None:
      self._hsState =  HsState(self)

    # Read in current state from the DB
    self._hsState.readStateFromDB()

    # This will hold the list of completed swarms that we find
    completedSwarms = set()

    # Mark the exhausted swarm as completing/completed, if any
    if exhaustedSwarmId is not None:
      self.logger.info("Removing swarm %s from the active set "
                       "because we can't find any new unique particle "
                       "positions" % (exhaustedSwarmId))
      # Is it completing or completed?
      (particles, _, _, _, _) = self._resultsDB.getParticleInfos(
                                      swarmId=exhaustedSwarmId, matured=False)
      if len(particles) > 0:
        exhaustedSwarmStatus = 'completing'
      else:
        exhaustedSwarmStatus = 'completed'

    # Kill all swarms that don't need to be explored based on the most recent
    # information.
    if self._killUselessSwarms:
      self._hsState.killUselessSwarms()

    # For all swarms that were in the 'completing' state, see if they have
    # completed yet.
    #
    # Note that we are not quite sure why this doesn't automatically get handled
    # when we receive notification that a model finally completed in a swarm.
    # But, we ARE running into a situation, when speculativeParticles is off,
    # where we have one or more swarms in the 'completing' state even though all
    # models have since finished. This logic will serve as a failsafe against
    # this situation.
    completingSwarms = self._hsState.getCompletingSwarms()
    for swarmId in completingSwarms:
      # Is it completed?
      (particles, _, _, _, _) = self._resultsDB.getParticleInfos(
                                      swarmId=swarmId, matured=False)
      if len(particles) == 0:
        completedSwarms.add(swarmId)

    # Are there any swarms we can remove (because they have matured)?
    completedSwarmGens = self._resultsDB.getMaturedSwarmGenerations()
    priorCompletedSwarms = self._hsState.getCompletedSwarms()
    for (swarmId, genIdx, errScore) in completedSwarmGens:

      # Don't need to report it if the swarm already completed
      if swarmId in priorCompletedSwarms:
        continue

      completedList = self._swarmTerminator.recordDataPoint(
          swarmId=swarmId, generation=genIdx, errScore=errScore)

      # Update status message
      statusMsg = "Completed generation #%d of swarm '%s' with a best" \
                  " errScore of %g" % (genIdx, swarmId, errScore)
      if len(completedList) > 0:
        statusMsg = "%s. Matured swarm(s): %s" % (statusMsg, completedList)
      self.logger.info(statusMsg)
      self._cjDAO.jobSetFields (jobID=self._jobID,
                                fields=dict(engStatus=statusMsg),
                                useConnectionID=False,
                                ignoreUnchanged=True)

      # Special test mode to check which swarms have terminated
      if 'NTA_TEST_recordSwarmTerminations' in os.environ:
        while True:
          resultsStr = self._cjDAO.jobGetFields(self._jobID, ['results'])[0]
          if resultsStr is None:
            results = {}
          else:
            results = json.loads(resultsStr)
          if not 'terminatedSwarms' in results:
            results['terminatedSwarms'] = {}

          for swarm in completedList:
            if swarm not in results['terminatedSwarms']:
              results['terminatedSwarms'][swarm] = (genIdx,
                                    self._swarmTerminator.swarmScores[swarm])

          newResultsStr = json.dumps(results)
          if newResultsStr == resultsStr:
            break
          updated = self._cjDAO.jobSetFieldIfEqual(jobID=self._jobID,
                                                   fieldName='results',
                                                   curValue=resultsStr,
                                                   newValue = json.dumps(results))
          if updated:
            break

      if len(completedList) > 0:
        for name in completedList:
          self.logger.info("Swarm matured: %s. Score at generation %d: "
                           "%s" % (name, genIdx, errScore))
        completedSwarms = completedSwarms.union(completedList)

    if len(completedSwarms)==0 and (exhaustedSwarmId is None):
      return

    # We need to mark one or more swarms as completed, keep trying until
    #  successful, or until some other worker does it for us.
    while True:

      if exhaustedSwarmId is not None:
        self._hsState.setSwarmState(exhaustedSwarmId, exhaustedSwarmStatus)

      # Mark the completed swarms as completed
      for swarmId in completedSwarms:
        self._hsState.setSwarmState(swarmId, 'completed')

      # If nothing changed, we're done
      if not self._hsState.isDirty():
        return

      # Update the shared Hypersearch state now
      # This will do nothing and return False if some other worker beat us to it
      success = self._hsState.writeStateToDB()

      if success:
        # Go through and cancel all models that are still running, except for
        # the best model. Once the best model changes, the one that used to be
        # best (and has  matured) will notice that and stop itself at that point.
        jobResultsStr = self._cjDAO.jobGetFields(self._jobID, ['results'])[0]
        if jobResultsStr is not None:
          jobResults = json.loads(jobResultsStr)
          bestModelId = jobResults.get('bestModel', None)
        else:
          bestModelId = None

        for swarmId in list(completedSwarms):
          (_, modelIds, _, _, _) = self._resultsDB.getParticleInfos(
                                          swarmId=swarmId, completed=False)
          if bestModelId in modelIds:
            modelIds.remove(bestModelId)
          if len(modelIds) == 0:
            continue
          self.logger.info("Killing the following models in swarm '%s' because"
                           "the swarm is being terminated: %s" % (swarmId,
                           str(modelIds)))

          for modelId in modelIds:
            self._cjDAO.modelSetFields(modelId,
                    dict(engStop=ClientJobsDAO.STOP_REASON_KILLED),
                    ignoreUnchanged = True)
        return

      # We were not able to change the state because some other worker beat us
      # to it.
      # Get the new state, and try again to apply our changes.
      self._hsState.readStateFromDB()
      self.logger.debug("New hsState has been set by some other worker to: "
                       " \n%s" % (pprint.pformat(self._hsState._state, indent=4)))