Instruction:
Can you tell what is the following Python 3 function doing
def _generateEncoderStringsV1(includedFields):

  # ------------------------------------------------------------------------
  # First accumulate the possible choices for each encoder
  encoderChoicesList = []
  for fieldInfo in includedFields:

    fieldName = fieldInfo['fieldName']

    # Get the list of encoder choices for this field
    (choicesList, aggFunction) = _generateEncoderChoicesV1(fieldInfo)
    encoderChoicesList.extend(choicesList)

  # ------------------------------------------------------------------------
  # Generate the string containing the encoder specs and encoder schema. See
  #  the function comments for an example of the encoderSpecsStr and
  #  encoderSchemaStr
  #
  encoderSpecsList = []
  for encoderChoices in encoderChoicesList:
    # Use the last choice as the default in the base file because the 1st is
    # often None
    encoder = encoderChoices[-1]

    # Check for bad characters
    for c in _ILLEGAL_FIELDNAME_CHARACTERS:
      if encoder['name'].find(c) >= 0:
        raise _ExpGeneratorException("Illegal character in field: %r (%r)" % (
          c, encoder['name']))

    encoderSpecsList.append("%s: \n%s%s" % (
        _quoteAndEscape(encoder['name']),
        2*_ONE_INDENT,
        pprint.pformat(encoder, indent=2*_INDENT_STEP)))

  encoderSpecsStr = ',\n  '.join(encoderSpecsList)

  # ------------------------------------------------------------------------
  # Generate the string containing the permutation encoder choices. See the
  #  function comments above for an example of the permEncoderChoicesStr

  permEncoderChoicesList = []
  for encoderChoices in encoderChoicesList:
    permEncoderChoicesList.append("%s: %s," % (
        _quoteAndEscape(encoderChoices[-1]['name']),
        pprint.pformat(encoderChoices, indent=2*_INDENT_STEP)))
  permEncoderChoicesStr = '\n'.join(permEncoderChoicesList)
  permEncoderChoicesStr = _indentLines(permEncoderChoicesStr, 1,
                                       indentFirstLine=False)

  # Return results
  return (encoderSpecsStr, permEncoderChoicesStr)

Response: Generates a string that contains the encoder related substitution variables for the base description file.


Instruction:
Write a Python 3 script for
generating the string that defines the permutations to apply for a given encoder.

Response:
def _generatePermEncoderStr(options, encoderDict):
  """ Generate the string that defines the permutations to apply for a given
  encoder.

  Parameters:
  -----------------------------------------------------------------------
  options: experiment params
  encoderDict: the encoder dict, which gets placed into the description.py

  For example, if the encoderDict contains:
      'consumption':     {
                'clipInput': True,
                'fieldname': u'consumption',
                'n': 100,
                'name': u'consumption',
                'type': 'AdaptiveScalarEncoder',
                'w': 21},

  The return string will contain:
    "PermuteEncoder(fieldName='consumption',
                    encoderClass='AdaptiveScalarEncoder',
                    w=21,
                    n=PermuteInt(28, 521),
                    clipInput=True)"

  """

  permStr = ""

  # If it's the encoder for the classifier input, then it's always present so
  # put it in as a dict in the permutations.py file instead of a
  # PermuteEncoder().
  if encoderDict.get('classifierOnly', False):
    permStr = "dict("
    for key, value in encoderDict.items():
      if key == "name":
        continue

      if key == 'n' and encoderDict['type'] != 'SDRCategoryEncoder':
        permStr += "n=PermuteInt(%d, %d), " % (encoderDict["w"] + 7,
                                               encoderDict["w"] + 500)
      else:
        if issubclass(type(value), basestring):
          permStr += "%s='%s', " % (key, value)
        else:
          permStr += "%s=%s, " % (key, value)
    permStr += ")"

  else:
    # Scalar encoders
    if encoderDict["type"] in ["ScalarSpaceEncoder", "AdaptiveScalarEncoder",
                             "ScalarEncoder", "LogEncoder"]:
      permStr = "PermuteEncoder("
      for key, value in encoderDict.items():
        if key == "fieldname":
          key = "fieldName"
        elif key == "type":
          key = "encoderClass"
        elif key == "name":
          continue

        if key == "n":
          permStr += "n=PermuteInt(%d, %d), " % (encoderDict["w"] + 1,
                                                 encoderDict["w"] + 500)
        elif key == "runDelta":
          if value and not "space" in encoderDict:
            permStr += "space=PermuteChoices([%s,%s]), " \
                     % (_quoteAndEscape("delta"), _quoteAndEscape("absolute"))
          encoderDict.pop("runDelta")

        else:
          if issubclass(type(value), basestring):
            permStr += "%s='%s', " % (key, value)
          else:
            permStr += "%s=%s, " % (key, value)
      permStr += ")"

    # Category encoder
    elif encoderDict["type"] in ["SDRCategoryEncoder"]:
      permStr = "PermuteEncoder("
      for key, value in encoderDict.items():
        if key == "fieldname":
          key = "fieldName"
        elif key == "type":
          key = "encoderClass"
        elif key == "name":
          continue

        if issubclass(type(value), basestring):
          permStr += "%s='%s', " % (key, value)
        else:
          permStr += "%s=%s, " % (key, value)
      permStr += ")"

    # Datetime encoder
    elif encoderDict["type"] in ["DateEncoder"]:
      permStr = "PermuteEncoder("
      for key, value in encoderDict.items():
        if key == "fieldname":
          key = "fieldName"
        elif key == "type":
          continue
        elif key == "name":
          continue

        if key == "timeOfDay":
          permStr += "encoderClass='%s.timeOfDay', " % (encoderDict["type"])
          permStr += "radius=PermuteFloat(0.5, 12), "
          permStr += "w=%d, " % (value[0])
        elif key == "dayOfWeek":
          permStr += "encoderClass='%s.dayOfWeek', " % (encoderDict["type"])
          permStr += "radius=PermuteFloat(1, 6), "
          permStr += "w=%d, " % (value[0])
        elif key == "weekend":
          permStr += "encoderClass='%s.weekend', " % (encoderDict["type"])
          permStr += "radius=PermuteChoices([1]),  "
          permStr += "w=%d, " % (value)
        else:
          if issubclass(type(value), basestring):
            permStr += "%s='%s', " % (key, value)
          else:
            permStr += "%s=%s, " % (key, value)
      permStr += ")"

    else:
      raise RuntimeError("Unsupported encoder type '%s'" % \
                          (encoderDict["type"]))

  return permStr


Instruction:
How would you code a function in Python 3 to
generate and returns the encoder string for the encoder in the version 2 of the file.

Response:
def _generateEncoderStringsV2(includedFields, options):
  """ Generate and return the following encoder related substitution variables:

  encoderSpecsStr:
    For the base description file, this string defines the default
    encoding dicts for each encoder. For example:

         __gym_encoder = {   'fieldname': 'gym',
          'n': 13,
          'name': 'gym',
          'type': 'SDRCategoryEncoder',
          'w': 7},
        __address_encoder = {   'fieldname': 'address',
          'n': 13,
          'name': 'address',
          'type': 'SDRCategoryEncoder',
          'w': 7}

  permEncoderChoicesStr:
    For the permutations file, this defines the possible
    encoder dicts for each encoder. For example:

        '__gym_encoder' : PermuteEncoder('gym', 'SDRCategoryEncoder', w=7,
            n=100),

        '__address_encoder' : PermuteEncoder('address', 'SDRCategoryEncoder',
              w=7, n=100),

        '__timestamp_dayOfWeek_encoder' : PermuteEncoder('timestamp',
            'DateEncoder.timeOfDay', w=7, radius=PermuteChoices([1, 8])),

        '__consumption_encoder': PermuteEncoder('consumption', 'AdaptiveScalarEncoder',
            w=7, n=PermuteInt(13, 500, 20), minval=0,
            maxval=PermuteInt(100, 300, 25)),

  Parameters:
  --------------------------------------------------
  includedFields:  item from the 'includedFields' section of the
                    description JSON object. This is a list of dicts, each
                    dict defining the field name, type, and optional min
                    and max values.

  retval:  (encoderSpecsStr permEncoderChoicesStr)

  """

  width = 21
  encoderDictsList = []

  # If this is a NontemporalClassification experiment, then the
  #  the "predicted" field (the classification value) should be marked to ONLY
  #  go to the classifier
  if options['inferenceType'] in ["NontemporalClassification",
                                  "NontemporalMultiStep",
                                  "TemporalMultiStep",
                                  "MultiStep"]:
    classifierOnlyField = options['inferenceArgs']['predictedField']
  else:
    classifierOnlyField = None

  # ==========================================================================
  # For each field, generate the default encoding dict and PermuteEncoder
  #  constructor arguments
  for fieldInfo in includedFields:

    fieldName = fieldInfo['fieldName']
    fieldType = fieldInfo['fieldType']

    # ---------
    # Scalar?
    if fieldType in ['float', 'int']:
      # n=100 is reasonably hardcoded value for n when used by description.py
      # The swarming will use PermuteEncoder below, where n is variable and
      # depends on w
      runDelta = fieldInfo.get("runDelta", False)
      if runDelta or "space" in fieldInfo:
        encoderDict = dict(type='ScalarSpaceEncoder', name=fieldName,
                            fieldname=fieldName, n=100, w=width, clipInput=True)
        if runDelta:
          encoderDict["runDelta"] = True
      else:
        encoderDict = dict(type='AdaptiveScalarEncoder', name=fieldName,
                            fieldname=fieldName, n=100, w=width, clipInput=True)

      if 'minValue' in fieldInfo:
        encoderDict['minval'] = fieldInfo['minValue']
      if 'maxValue' in fieldInfo:
        encoderDict['maxval'] = fieldInfo['maxValue']

      # If both min and max were specified, use a non-adaptive encoder
      if ('minValue' in fieldInfo and 'maxValue' in fieldInfo) \
            and (encoderDict['type'] == 'AdaptiveScalarEncoder'):
        encoderDict['type'] = 'ScalarEncoder'

      # Defaults may have been over-ridden by specifying an encoder type
      if 'encoderType' in fieldInfo:
        encoderDict['type'] = fieldInfo['encoderType']

      if 'space' in fieldInfo:
        encoderDict['space'] = fieldInfo['space']
      encoderDictsList.append(encoderDict)

    # ---------
    # String?
    elif fieldType == 'string':
      encoderDict = dict(type='SDRCategoryEncoder', name=fieldName,
                     fieldname=fieldName, n=100+width, w=width)
      if 'encoderType' in fieldInfo:
        encoderDict['type'] = fieldInfo['encoderType']

      encoderDictsList.append(encoderDict)

    # ---------
    # Datetime?
    elif fieldType == 'datetime':

      # First, the time of day representation
      encoderDict = dict(type='DateEncoder', name='%s_timeOfDay' % (fieldName),
                     fieldname=fieldName, timeOfDay=(width, 1))
      if 'encoderType' in fieldInfo:
        encoderDict['type'] = fieldInfo['encoderType']
      encoderDictsList.append(encoderDict)

      # Now, the day of week representation
      encoderDict = dict(type='DateEncoder', name='%s_dayOfWeek' % (fieldName),
                     fieldname=fieldName, dayOfWeek=(width, 1))
      if 'encoderType' in fieldInfo:
        encoderDict['type'] = fieldInfo['encoderType']
      encoderDictsList.append(encoderDict)

      # Now, the day of week representation
      encoderDict = dict(type='DateEncoder', name='%s_weekend' % (fieldName),
                     fieldname=fieldName, weekend=(width))
      if 'encoderType' in fieldInfo:
        encoderDict['type'] = fieldInfo['encoderType']
      encoderDictsList.append(encoderDict)

    else:
      raise RuntimeError("Unsupported field type '%s'" % (fieldType))

    # -----------------------------------------------------------------------
    # If this was the predicted field, insert another encoder that sends it
    # to the classifier only
    if fieldName == classifierOnlyField:
      clEncoderDict = dict(encoderDict)
      clEncoderDict['classifierOnly'] = True
      clEncoderDict['name'] = '_classifierInput'
      encoderDictsList.append(clEncoderDict)

      # If the predicted field needs to be excluded, take it out of the encoder
      #  lists
      if options["inferenceArgs"]["inputPredictedField"] == "no":
        encoderDictsList.remove(encoderDict)

  # Remove any encoders not in fixedFields
  if options.get('fixedFields') is not None:
    tempList=[]
    for encoderDict in encoderDictsList:
      if encoderDict['name'] in options['fixedFields']:
        tempList.append(encoderDict)
    encoderDictsList = tempList

  # ==========================================================================
  # Now generate the encoderSpecsStr and permEncoderChoicesStr strings from
  #  encoderDictsList and constructorStringList

  encoderSpecsList = []
  permEncoderChoicesList = []
  for encoderDict in encoderDictsList:

    if encoderDict['name'].find('\\') >= 0:
      raise _ExpGeneratorException("Illegal character in field: '\\'")

    # Check for bad characters
    for c in _ILLEGAL_FIELDNAME_CHARACTERS:
      if encoderDict['name'].find(c) >= 0:
        raise _ExpGeneratorException("Illegal character %s in field %r"  %(c, encoderDict['name']))

    constructorStr = _generatePermEncoderStr(options, encoderDict)

    encoderKey = _quoteAndEscape(encoderDict['name'])
    encoderSpecsList.append("%s: %s%s" % (
        encoderKey,
        2*_ONE_INDENT,
        pprint.pformat(encoderDict, indent=2*_INDENT_STEP)))

    # Each permEncoderChoicesStr is of the form:
    #  PermuteEncoder('gym', 'SDRCategoryEncoder',
    #          w=7, n=100),
    permEncoderChoicesList.append("%s: %s," % (encoderKey, constructorStr))

  # Join into strings
  encoderSpecsStr = ',\n  '.join(encoderSpecsList)

  permEncoderChoicesStr = '\n'.join(permEncoderChoicesList)
  permEncoderChoicesStr = _indentLines(permEncoderChoicesStr, 1,
                                       indentFirstLine=True)

  # Return results
  return (encoderSpecsStr, permEncoderChoicesStr)


Instruction:
Can you write a function in Python 3 where it
handles legacy options for JAVA parameters.

Response:
def _handleJAVAParameters(options):
  """ Handle legacy options (TEMPORARY) """

  # Find the correct InferenceType for the Model
  if 'inferenceType' not in options:
    prediction = options.get('prediction', {InferenceType.TemporalNextStep:
                                              {'optimize':True}})
    inferenceType = None
    for infType, value in prediction.iteritems():
      if value['optimize']:
        inferenceType = infType
        break

    if inferenceType == 'temporal':
      inferenceType = InferenceType.TemporalNextStep
    if inferenceType != InferenceType.TemporalNextStep:
      raise _ExpGeneratorException("Unsupported inference type %s"  % \
                                    (inferenceType))
    options['inferenceType'] = inferenceType

  # Find the best value for the predicted field
  if 'predictionField' in options:
    if 'inferenceArgs' not in options:
      options['inferenceArgs'] = {'predictedField': options['predictionField']}
    elif 'predictedField' not in options['inferenceArgs']:
      options['inferenceArgs']['predictedField'] = options['predictionField']


Instruction:
How would you explain what the following Python 3 function does
def _getPropertyValue(schema, propertyName, options):

  if propertyName not in options:
    paramsSchema = schema['properties'][propertyName]
    if 'default' in paramsSchema:
      options[propertyName] = paramsSchema['default']
    else:
      options[propertyName] = None

Response: Checks to see if property is specified in options.


Instruction:
Explain what the following Python 3 code does
def _getExperimentDescriptionSchema():
  installPath = os.path.dirname(os.path.abspath(__file__))
  schemaFilePath = os.path.join(installPath, "experimentDescriptionSchema.json")
  return json.loads(open(schemaFilePath, 'r').read())

Response: Returns the experiment description schema.


Instruction:
Can you tell what is the following Python 3 function doing
def _generateExperiment(options, outputDirPath, hsVersion,
                             claDescriptionTemplateFile):
  """ Executes the --description option, which includes:

      1. Perform provider compatibility checks
      2. Preprocess the training and testing datasets (filter, join providers)
      3. If test dataset omitted, split the training dataset into training
         and testing datasets.
      4. Gather statistics about the training and testing datasets.
      5. Generate experiment scripts (description.py, permutaions.py)

  Parameters:
  --------------------------------------------------------------------------
  options:  dictionary that matches the schema defined by the return value of
            _getExperimentDescriptionSchema();  NOTE: this arg may be modified
            by this function.

  outputDirPath:  where to place generated files

  hsVersion:  which version of hypersearch permutations file to generate, can
                be 'v1' or 'v2'
  claDescriptionTemplateFile: Filename containing the template description

  Returns:    on success, returns a dictionary per _experimentResultsJSONSchema;
              raises exception on error

      Assumption1: input train and test files have identical field metadata
  """

  _gExperimentDescriptionSchema = _getExperimentDescriptionSchema()

  # Validate JSON arg using JSON schema validator
  try:
    validictory.validate(options, _gExperimentDescriptionSchema)
  except Exception, e:
    raise _InvalidCommandArgException(
      ("JSON arg validation failed for option --description: " + \
       "%s\nOPTION ARG=%s") % (str(e), pprint.pformat(options)))

  # Validate the streamDef
  streamSchema = json.load(resource_stream(jsonschema.__name__,
                                           'stream_def.json'))
  try:
    validictory.validate(options['streamDef'], streamSchema)
  except Exception, e:
    raise _InvalidCommandArgException(
      ("JSON arg validation failed for streamDef " + \
       "%s\nOPTION ARG=%s") % (str(e), json.dumps(options)))

  # -----------------------------------------------------------------------
  # Handle legacy parameters from JAVA API server
  # TODO: remove this!
  _handleJAVAParameters(options)

  # -----------------------------------------------------------------------
  # Get default values
  for propertyName in _gExperimentDescriptionSchema['properties']:
    _getPropertyValue(_gExperimentDescriptionSchema, propertyName, options)

  if options['inferenceArgs'] is not None:
    infArgs = _gExperimentDescriptionSchema['properties']['inferenceArgs']
    for schema in infArgs['type']:
      if isinstance(schema, dict):
        for propertyName in schema['properties']:
          _getPropertyValue(schema, propertyName, options['inferenceArgs'])

  if options['anomalyParams'] is not None:
    anomalyArgs = _gExperimentDescriptionSchema['properties']['anomalyParams']
    for schema in anomalyArgs['type']:
      if isinstance(schema, dict):
        for propertyName in schema['properties']:
          _getPropertyValue(schema, propertyName, options['anomalyParams'])

  # If the user specified nonTemporalClassification, make sure prediction
  # steps is 0
  predictionSteps = options['inferenceArgs'].get('predictionSteps', None)
  if options['inferenceType'] == InferenceType.NontemporalClassification:
    if predictionSteps is not None and predictionSteps != [0]:
      raise RuntimeError("When NontemporalClassification is used, prediction"
                         " steps must be [0]")

  # -------------------------------------------------------------------------
  # If the user asked for 0 steps of prediction, then make this a spatial
  #  classification experiment
  if predictionSteps == [0] \
    and options['inferenceType'] in ['NontemporalMultiStep',
                                     'TemporalMultiStep',
                                     'MultiStep']:
    options['inferenceType'] = InferenceType.NontemporalClassification

  # If NontemporalClassification was chosen as the inferenceType, then the
  #  predicted field can NOT be used as an input
  if options["inferenceType"] == InferenceType.NontemporalClassification:
    if options["inferenceArgs"]["inputPredictedField"] == "yes" \
        or options["inferenceArgs"]["inputPredictedField"] == "auto":
      raise RuntimeError("When the inference type is NontemporalClassification"
                         " inputPredictedField must be set to 'no'")
    options["inferenceArgs"]["inputPredictedField"] = "no"

  # -----------------------------------------------------------------------
  # Process the swarmSize setting, if provided
  swarmSize = options['swarmSize']

  if swarmSize is None:
    if options["inferenceArgs"]["inputPredictedField"] is None:
      options["inferenceArgs"]["inputPredictedField"] = "auto"

  elif swarmSize == 'small':
    if options['minParticlesPerSwarm'] is None:
      options['minParticlesPerSwarm'] = 3
    if options['iterationCount'] is None:
      options['iterationCount'] = 100
    if options['maxModels'] is None:
      options['maxModels'] = 1
    if options["inferenceArgs"]["inputPredictedField"] is None:
      options["inferenceArgs"]["inputPredictedField"] = "yes"

  elif swarmSize == 'medium':
    if options['minParticlesPerSwarm'] is None:
      options['minParticlesPerSwarm'] = 5
    if options['iterationCount'] is None:
      options['iterationCount'] = 4000
    if options['maxModels'] is None:
      options['maxModels'] = 200
    if options["inferenceArgs"]["inputPredictedField"] is None:
      options["inferenceArgs"]["inputPredictedField"] = "auto"

  elif swarmSize == 'large':
    if options['minParticlesPerSwarm'] is None:
      options['minParticlesPerSwarm'] = 15
    #options['killUselessSwarms'] = False
    #options['minFieldContribution'] = -1000
    #options['maxFieldBranching'] = 10
    #options['tryAll3FieldCombinations'] = True
    options['tryAll3FieldCombinationsWTimestamps'] = True
    if options["inferenceArgs"]["inputPredictedField"] is None:
      options["inferenceArgs"]["inputPredictedField"] = "auto"

  else:
    raise RuntimeError("Unsupported swarm size: %s" % (swarmSize))

  # -----------------------------------------------------------------------
  # Get token replacements
  tokenReplacements = dict()

  #--------------------------------------------------------------------------
  # Generate the encoder related substitution strings
  includedFields = options['includedFields']
  if hsVersion == 'v1':
    (encoderSpecsStr, permEncoderChoicesStr) = \
        _generateEncoderStringsV1(includedFields)
  elif hsVersion in ['v2', 'ensemble']:
    (encoderSpecsStr, permEncoderChoicesStr) = \
        _generateEncoderStringsV2(includedFields, options)
  else:
    raise RuntimeError("Unsupported hsVersion of %s" % (hsVersion))

  #--------------------------------------------------------------------------
  # Generate the string containing the sensor auto-reset dict.
  if options['resetPeriod'] is not None:
    sensorAutoResetStr = pprint.pformat(options['resetPeriod'],
                                         indent=2*_INDENT_STEP)
  else:
    sensorAutoResetStr = 'None'

  #--------------------------------------------------------------------------
  # Generate the string containing the aggregation settings.
  aggregationPeriod = {
      'days': 0,
      'hours': 0,
      'microseconds': 0,
      'milliseconds': 0,
      'minutes': 0,
      'months': 0,
      'seconds': 0,
      'weeks': 0,
      'years': 0,
  }

  # Honor any overrides provided in the stream definition
  aggFunctionsDict = {}
  if 'aggregation' in options['streamDef']:
    for key in aggregationPeriod.keys():
      if key in options['streamDef']['aggregation']:
        aggregationPeriod[key] = options['streamDef']['aggregation'][key]
    if 'fields' in options['streamDef']['aggregation']:
      for (fieldName, func) in options['streamDef']['aggregation']['fields']:
        aggFunctionsDict[fieldName] = str(func)

  # Do we have any aggregation at all?
  hasAggregation = False
  for v in aggregationPeriod.values():
    if v != 0:
      hasAggregation = True
      break

  # Convert the aggFunctionsDict to a list
  aggFunctionList = aggFunctionsDict.items()
  aggregationInfo = dict(aggregationPeriod)
  aggregationInfo['fields'] = aggFunctionList

  # Form the aggregation strings
  aggregationInfoStr = "%s" % (pprint.pformat(aggregationInfo,
                                              indent=2*_INDENT_STEP))

  # -----------------------------------------------------------------------
  # Generate the string defining the dataset. This is basically the
  #  streamDef, but referencing the aggregation we already pulled out into the
  #  config dict (which enables permuting over it)
  datasetSpec = options['streamDef']
  if 'aggregation' in datasetSpec:
    datasetSpec.pop('aggregation')
  if hasAggregation:
    datasetSpec['aggregation'] = '$SUBSTITUTE'
  datasetSpecStr = pprint.pformat(datasetSpec, indent=2*_INDENT_STEP)
  datasetSpecStr = datasetSpecStr.replace(
      "'$SUBSTITUTE'", "config['aggregationInfo']")
  datasetSpecStr = _indentLines(datasetSpecStr, 2, indentFirstLine=False)

  # -----------------------------------------------------------------------
  # Was computeInterval specified with Multistep prediction? If so, this swarm
  #  should permute over different aggregations
  computeInterval = options['computeInterval']
  if computeInterval is not None \
      and options['inferenceType'] in ['NontemporalMultiStep',
                                       'TemporalMultiStep',
                                       'MultiStep']:

    # Compute the predictAheadTime based on the minAggregation (specified in
    #  the stream definition) and the number of prediction steps
    predictionSteps = options['inferenceArgs'].get('predictionSteps', [1])
    if len(predictionSteps) > 1:
      raise _InvalidCommandArgException("Invalid predictionSteps: %s. " \
              "When computeInterval is specified, there can only be one " \
              "stepSize in predictionSteps." % predictionSteps)

    if max(aggregationInfo.values()) == 0:
      raise _InvalidCommandArgException("Missing or nil stream aggregation: "
            "When computeInterval is specified, then the stream aggregation "
            "interval must be non-zero.")

    # Compute the predictAheadTime
    numSteps = predictionSteps[0]
    predictAheadTime = dict(aggregationPeriod)
    for key in predictAheadTime.iterkeys():
      predictAheadTime[key] *= numSteps
    predictAheadTimeStr = pprint.pformat(predictAheadTime,
                                         indent=2*_INDENT_STEP)

    # This tells us to plug in a wildcard string for the prediction steps that
    #  we use in other parts of the description file (metrics, inferenceArgs,
    #  etc.)
    options['dynamicPredictionSteps'] = True

  else:
    options['dynamicPredictionSteps'] = False
    predictAheadTimeStr = "None"

  # -----------------------------------------------------------------------
  # Save environment-common token substitutions

  tokenReplacements['\$EXP_GENERATOR_PROGRAM_PATH'] = \
                                _quoteAndEscape(os.path.abspath(__file__))

  # If the "uber" metric 'MultiStep' was specified, then plug in TemporalMultiStep
  #  by default
  inferenceType = options['inferenceType']
  if inferenceType == 'MultiStep':
    inferenceType = InferenceType.TemporalMultiStep
  tokenReplacements['\$INFERENCE_TYPE'] = "'%s'" % inferenceType

  # Nontemporal classificaion uses only encoder and classifier
  if inferenceType == InferenceType.NontemporalClassification:
    tokenReplacements['\$SP_ENABLE'] = "False"
    tokenReplacements['\$TP_ENABLE'] = "False"
  else:
    tokenReplacements['\$SP_ENABLE'] = "True"
    tokenReplacements['\$TP_ENABLE'] = "True"
    tokenReplacements['\$CLA_CLASSIFIER_IMPL'] = ""

  tokenReplacements['\$ANOMALY_PARAMS'] = pprint.pformat(
      options['anomalyParams'], indent=2*_INDENT_STEP)

  tokenReplacements['\$ENCODER_SPECS'] = encoderSpecsStr
  tokenReplacements['\$SENSOR_AUTO_RESET'] = sensorAutoResetStr

  tokenReplacements['\$AGGREGATION_INFO'] = aggregationInfoStr

  tokenReplacements['\$DATASET_SPEC'] = datasetSpecStr
  if options['iterationCount'] is None:
    options['iterationCount'] = -1
  tokenReplacements['\$ITERATION_COUNT'] \
                                        = str(options['iterationCount'])

  tokenReplacements['\$SP_POOL_PCT'] \
                                        = str(options['spCoincInputPoolPct'])

  tokenReplacements['\$HS_MIN_PARTICLES'] \
                                        = str(options['minParticlesPerSwarm'])

  tokenReplacements['\$SP_PERM_CONNECTED'] \
                                        = str(options['spSynPermConnected'])

  tokenReplacements['\$FIELD_PERMUTATION_LIMIT'] \
                                        = str(options['fieldPermutationLimit'])

  tokenReplacements['\$PERM_ENCODER_CHOICES'] \
                                        = permEncoderChoicesStr

  predictionSteps = options['inferenceArgs'].get('predictionSteps', [1])
  predictionStepsStr = ','.join([str(x) for x in predictionSteps])
  tokenReplacements['\$PREDICTION_STEPS'] = "'%s'" % (predictionStepsStr)

  tokenReplacements['\$PREDICT_AHEAD_TIME'] = predictAheadTimeStr

  # Option permuting over SP synapse decrement value
  tokenReplacements['\$PERM_SP_CHOICES'] = ""
  if options['spPermuteDecrement'] \
        and options['inferenceType'] != 'NontemporalClassification':
    tokenReplacements['\$PERM_SP_CHOICES'] = \
      _ONE_INDENT +"'synPermInactiveDec': PermuteFloat(0.0003, 0.1),\n"

  # The TM permutation parameters are not required for non-temporal networks
  if options['inferenceType'] in ['NontemporalMultiStep',
                                  'NontemporalClassification']:
    tokenReplacements['\$PERM_TP_CHOICES'] = ""
  else:
    tokenReplacements['\$PERM_TP_CHOICES'] = \
        "  'activationThreshold': PermuteInt(12, 16),\n"  \
      + "  'minThreshold': PermuteInt(9, 12),\n" \
      + "  'pamLength': PermuteInt(1, 5),\n"

  # If the inference type is just the generic 'MultiStep', then permute over
  #  temporal/nonTemporal multistep
  if options['inferenceType'] == 'MultiStep':
    tokenReplacements['\$PERM_INFERENCE_TYPE_CHOICES'] = \
      "  'inferenceType': PermuteChoices(['NontemporalMultiStep', " \
      + "'TemporalMultiStep']),"
  else:
    tokenReplacements['\$PERM_INFERENCE_TYPE_CHOICES'] = ""

  # The Classifier permutation parameters are only required for
  #  Multi-step inference types
  if options['inferenceType'] in ['NontemporalMultiStep', 'TemporalMultiStep',
                                  'MultiStep', 'TemporalAnomaly',
                                  'NontemporalClassification']:
    tokenReplacements['\$PERM_CL_CHOICES'] = \
        "  'alpha': PermuteFloat(0.0001, 0.1),\n"

  else:
    tokenReplacements['\$PERM_CL_CHOICES'] = ""

  # The Permutations alwaysIncludePredictedField setting.
  # * When the experiment description has 'inputPredictedField' set to 'no', we
  #   simply do not put in an encoder for the predicted field.
  # * When 'inputPredictedField' is set to 'auto', we include an encoder for the
  #   predicted field and swarming tries it out just like all the other fields.
  # * When 'inputPredictedField' is set to 'yes', we include this setting in
  #   the permutations file which informs swarming to always use the
  #   predicted field (the first swarm will be the predicted field only)
  tokenReplacements['\$PERM_ALWAYS_INCLUDE_PREDICTED_FIELD'] = \
      "inputPredictedField = '%s'" % \
                            (options["inferenceArgs"]["inputPredictedField"])

  # The Permutations minFieldContribution setting
  if options.get('minFieldContribution', None) is not None:
    tokenReplacements['\$PERM_MIN_FIELD_CONTRIBUTION'] = \
        "minFieldContribution = %d" % (options['minFieldContribution'])
  else:
    tokenReplacements['\$PERM_MIN_FIELD_CONTRIBUTION'] = ""

  # The Permutations killUselessSwarms setting
  if options.get('killUselessSwarms', None) is not None:
    tokenReplacements['\$PERM_KILL_USELESS_SWARMS'] = \
        "killUselessSwarms = %r" % (options['killUselessSwarms'])
  else:
    tokenReplacements['\$PERM_KILL_USELESS_SWARMS'] = ""

  # The Permutations maxFieldBranching setting
  if options.get('maxFieldBranching', None) is not None:
    tokenReplacements['\$PERM_MAX_FIELD_BRANCHING'] = \
        "maxFieldBranching = %r" % (options['maxFieldBranching'])
  else:
    tokenReplacements['\$PERM_MAX_FIELD_BRANCHING'] = ""

  # The Permutations tryAll3FieldCombinations setting
  if options.get('tryAll3FieldCombinations', None) is not None:
    tokenReplacements['\$PERM_TRY_ALL_3_FIELD_COMBINATIONS'] = \
        "tryAll3FieldCombinations = %r" % (options['tryAll3FieldCombinations'])
  else:
    tokenReplacements['\$PERM_TRY_ALL_3_FIELD_COMBINATIONS'] = ""

  # The Permutations tryAll3FieldCombinationsWTimestamps setting
  if options.get('tryAll3FieldCombinationsWTimestamps', None) is not None:
    tokenReplacements['\$PERM_TRY_ALL_3_FIELD_COMBINATIONS_W_TIMESTAMPS'] = \
        "tryAll3FieldCombinationsWTimestamps = %r" % \
                (options['tryAll3FieldCombinationsWTimestamps'])
  else:
    tokenReplacements['\$PERM_TRY_ALL_3_FIELD_COMBINATIONS_W_TIMESTAMPS'] = ""

  # The Permutations fieldFields setting
  if options.get('fixedFields', None) is not None:
    tokenReplacements['\$PERM_FIXED_FIELDS'] = \
        "fixedFields = %r" % (options['fixedFields'])
  else:
    tokenReplacements['\$PERM_FIXED_FIELDS'] = ""

  # The Permutations fastSwarmModelParams setting
  if options.get('fastSwarmModelParams', None) is not None:
    tokenReplacements['\$PERM_FAST_SWARM_MODEL_PARAMS'] = \
        "fastSwarmModelParams = %r" % (options['fastSwarmModelParams'])
  else:
    tokenReplacements['\$PERM_FAST_SWARM_MODEL_PARAMS'] = ""

  # The Permutations maxModels setting
  if options.get('maxModels', None) is not None:
    tokenReplacements['\$PERM_MAX_MODELS'] = \
        "maxModels = %r" % (options['maxModels'])
  else:
    tokenReplacements['\$PERM_MAX_MODELS'] = ""

  # --------------------------------------------------------------------------
  # The Aggregation choices have to be determined when we are permuting over
  #   aggregations.
  if options['dynamicPredictionSteps']:
    debugAgg = True

    # First, we need to error check to insure that computeInterval is an integer
    #  multiple of minAggregation (aggregationPeriod)
    quotient = aggregationDivide(computeInterval, aggregationPeriod)
    (isInt, multiple) = _isInt(quotient)
    if not isInt or multiple < 1:
      raise _InvalidCommandArgException("Invalid computeInterval: %s. "
              "computeInterval must be an integer multiple of the stream "
              "aggregation (%s)." % (computeInterval, aggregationPeriod))

    # The valid aggregation choices are governed by the following constraint,
    #   1.) (minAggregation * N) * M = predictAheadTime
    #       (minAggregation * N) * M = maxPredictionSteps * minAggregation
    #       N * M = maxPredictionSteps
    #
    #   2.) computeInterval = K * aggregation
    #       computeInterval = K * (minAggregation * N)
    #
    # where: aggregation = minAggregation * N
    #        K, M and N are integers >= 1
    #          N = aggregation / minAggregation
    #          M = predictionSteps, for a particular aggregation
    #          K = number of predictions within each compute interval
    #

    # Let's build up a a list of the possible N's that satisfy the
    #  N * M = maxPredictionSteps constraint
    mTimesN = float(predictionSteps[0])
    possibleNs = []
    for n in xrange(1, int(mTimesN)+1):
      m = mTimesN / n
      mInt = int(round(m))
      if mInt < 1:
        break
      if abs(m - mInt) > 0.0001 * m:
        continue
      possibleNs.append(n)

    if debugAgg:
      print "All integer factors of %d are: %s" % (mTimesN, possibleNs)

    # Now go through and throw out any N's that don't satisfy the constraint:
    #  computeInterval = K * (minAggregation * N)
    aggChoices = []
    for n in possibleNs:
      # Compute minAggregation * N
      agg = dict(aggregationPeriod)
      for key in agg.iterkeys():
        agg[key] *= n

      # Make sure computeInterval is an integer multiple of the aggregation
      # period
      quotient = aggregationDivide(computeInterval, agg)
      #print computeInterval, agg
      #print quotient
      #import sys; sys.exit()
      (isInt, multiple) = _isInt(quotient)
      if not isInt or multiple < 1:
        continue
      aggChoices.append(agg)

    # Only eveluate up to 5 different aggregations
    aggChoices = aggChoices[-5:]

    if debugAgg:
      print "Aggregation choices that will be evaluted during swarming:"
      for agg in aggChoices:
        print "  ==>", agg
      print

    tokenReplacements['\$PERM_AGGREGATION_CHOICES'] = (
        "PermuteChoices(%s)" % (
            pprint.pformat(aggChoices, indent=2*_INDENT_STEP)))

  else:
    tokenReplacements['\$PERM_AGGREGATION_CHOICES'] = aggregationInfoStr

  # Generate the inferenceArgs replacement tokens
  _generateInferenceArgs(options, tokenReplacements)

  # Generate the metric replacement tokens
  _generateMetricsSubstitutions(options, tokenReplacements)

  # -----------------------------------------------------------------------
  # Generate Control dictionary
  environment = options['environment']
  if environment == OpfEnvironment.Nupic:
    tokenReplacements['\$ENVIRONMENT'] = "'%s'"%OpfEnvironment.Nupic
    controlTemplate = "nupicEnvironmentTemplate.tpl"
  elif environment == OpfEnvironment.Experiment:
    tokenReplacements['\$ENVIRONMENT'] = "'%s'"%OpfEnvironment.Experiment
    controlTemplate = "opfExperimentTemplate.tpl"
  else:
    raise _InvalidCommandArgException("Invalid environment type %s"% environment)

  # -----------------------------------------------------------------------
  if outputDirPath is None:
    outputDirPath = tempfile.mkdtemp()
  if not os.path.exists(outputDirPath):
    os.makedirs(outputDirPath)

  print "Generating experiment files in directory: %s..." % (outputDirPath)
  descriptionPyPath = os.path.join(outputDirPath, "description.py")
  _generateFileFromTemplates([claDescriptionTemplateFile, controlTemplate],
                              descriptionPyPath,
                              tokenReplacements)

  permutationsPyPath = os.path.join(outputDirPath, "permutations.py")

  if hsVersion == 'v1':
    _generateFileFromTemplates(['permutationsTemplateV1.tpl'],permutationsPyPath,
                            tokenReplacements)
  elif hsVersion == 'ensemble':
    _generateFileFromTemplates(['permutationsTemplateEnsemble.tpl'],permutationsPyPath,
                            tokenReplacements)
  elif hsVersion == 'v2':
    _generateFileFromTemplates(['permutationsTemplateV2.tpl'],permutationsPyPath,
                            tokenReplacements)
  else:
    raise(ValueError("This permutation version is not supported yet: %s" %
                        hsVersion))

  print "done."

Response: Generates the experiment for the given options.


Instruction:
Create a Python 3 function to
generate the token substitution for the metrics related fields.

Response:
def _generateMetricsSubstitutions(options, tokenReplacements):
  """Generate the token substitution for metrics related fields.
  This includes:
    \$METRICS
    \$LOGGED_METRICS
    \$PERM_OPTIMIZE_SETTING
  """
  # -----------------------------------------------------------------------
  #
  options['loggedMetrics'] = [".*"]

  # -----------------------------------------------------------------------
  # Generate the required metrics
  metricList, optimizeMetricLabel = _generateMetricSpecs(options)

  metricListString = ",\n".join(metricList)
  metricListString = _indentLines(metricListString, 2, indentFirstLine=False)
  permOptimizeSettingStr = 'minimize = "%s"' % optimizeMetricLabel
  # -----------------------------------------------------------------------
  # Specify which metrics should be logged
  loggedMetricsListAsStr = "[%s]" % (", ".join(["'%s'"% ptrn
                                              for ptrn in options['loggedMetrics']]))

  tokenReplacements['\$LOGGED_METRICS'] \
                                        = loggedMetricsListAsStr

  tokenReplacements['\$METRICS'] = metricListString

  tokenReplacements['\$PERM_OPTIMIZE_SETTING'] \
                                        = permOptimizeSettingStr


Instruction:
Explain what the following Python 3 code does
def _generateMetricSpecs(options):
  inferenceType = options['inferenceType']
  inferenceArgs = options['inferenceArgs']
  predictionSteps = inferenceArgs['predictionSteps']
  metricWindow = options['metricWindow']
  if metricWindow is None:
    metricWindow = int(Configuration.get("nupic.opf.metricWindow"))

  metricSpecStrings = []
  optimizeMetricLabel = ""

  # -----------------------------------------------------------------------
  # Generate the metrics specified by the expGenerator paramters
  metricSpecStrings.extend(_generateExtraMetricSpecs(options))

  # -----------------------------------------------------------------------

  optimizeMetricSpec = None
  # If using a dynamically computed prediction steps (i.e. when swarming
  #  over aggregation is requested), then we will plug in the variable
  #  predictionSteps in place of the statically provided predictionSteps
  #  from the JSON description.
  if options['dynamicPredictionSteps']:
    assert len(predictionSteps) == 1
    predictionSteps = ['$REPLACE_ME']

  # -----------------------------------------------------------------------
  # Metrics for temporal prediction
  if inferenceType in (InferenceType.TemporalNextStep,
                       InferenceType.TemporalAnomaly,
                       InferenceType.TemporalMultiStep,
                       InferenceType.NontemporalMultiStep,
                       InferenceType.NontemporalClassification,
                       'MultiStep'):

    predictedFieldName, predictedFieldType = _getPredictedField(options)
    isCategory = _isCategory(predictedFieldType)
    metricNames = ('avg_err',) if isCategory else ('aae', 'altMAPE')
    trivialErrorMetric = 'avg_err' if isCategory else 'altMAPE'
    oneGramErrorMetric = 'avg_err' if isCategory else 'altMAPE'
    movingAverageBaselineName = 'moving_mode' if isCategory else 'moving_mean'

    # Multi-step metrics
    for metricName in metricNames:
      metricSpec, metricLabel = \
        _generateMetricSpecString(field=predictedFieldName,
                 inferenceElement=InferenceElement.multiStepBestPredictions,
                 metric='multiStep',
                 params={'errorMetric': metricName,
                               'window':metricWindow,
                               'steps': predictionSteps},
                 returnLabel=True)
      metricSpecStrings.append(metricSpec)

    # If the custom error metric was specified, add that
    if options["customErrorMetric"] is not None :
      metricParams = dict(options["customErrorMetric"])
      metricParams['errorMetric'] = 'custom_error_metric'
      metricParams['steps'] = predictionSteps
      # If errorWindow is not specified, make it equal to the default window
      if not "errorWindow" in metricParams:
        metricParams["errorWindow"] = metricWindow
      metricSpec, metricLabel =_generateMetricSpecString(field=predictedFieldName,
                   inferenceElement=InferenceElement.multiStepPredictions,
                   metric="multiStep",
                   params=metricParams,
                   returnLabel=True)
      metricSpecStrings.append(metricSpec)

    # If this is the first specified step size, optimize for it. Be sure to
    #  escape special characters since this is a regular expression
    optimizeMetricSpec = metricSpec
    metricLabel = metricLabel.replace('[', '\\[')
    metricLabel = metricLabel.replace(']', '\\]')
    optimizeMetricLabel = metricLabel

    if options["customErrorMetric"] is not None :
      optimizeMetricLabel = ".*custom_error_metric.*"

    # Add in the trivial metrics
    if options["runBaselines"] \
          and inferenceType != InferenceType.NontemporalClassification:
      for steps in predictionSteps:
        metricSpecStrings.append(
          _generateMetricSpecString(field=predictedFieldName,
                                    inferenceElement=InferenceElement.prediction,
                                    metric="trivial",
                                    params={'window':metricWindow,
                                                  "errorMetric":trivialErrorMetric,
                                                  'steps': steps})
          )

        ##Add in the One-Gram baseline error metric
        #metricSpecStrings.append(
        #  _generateMetricSpecString(field=predictedFieldName,
        #                            inferenceElement=InferenceElement.encodings,
        #                            metric="two_gram",
        #                            params={'window':metricWindow,
        #                                          "errorMetric":oneGramErrorMetric,
        #                                          'predictionField':predictedFieldName,
        #                                          'steps': steps})
        #  )
        #
        #Include the baseline moving mean/mode metric
        if isCategory:
          metricSpecStrings.append(
            _generateMetricSpecString(field=predictedFieldName,
                                      inferenceElement=InferenceElement.prediction,
                                      metric=movingAverageBaselineName,
                                      params={'window':metricWindow
                                                    ,"errorMetric":"avg_err",
                                                    "mode_window":200,
                                                    "steps": steps})
            )
        else :
          metricSpecStrings.append(
            _generateMetricSpecString(field=predictedFieldName,
                                      inferenceElement=InferenceElement.prediction,
                                      metric=movingAverageBaselineName,
                                      params={'window':metricWindow
                                                    ,"errorMetric":"altMAPE",
                                                    "mean_window":200,
                                                    "steps": steps})
            )

  # -----------------------------------------------------------------------
  # Metrics for classification
  elif inferenceType in (InferenceType.TemporalClassification):

    metricName = 'avg_err'
    trivialErrorMetric = 'avg_err'
    oneGramErrorMetric = 'avg_err'
    movingAverageBaselineName = 'moving_mode'

    optimizeMetricSpec, optimizeMetricLabel = \
      _generateMetricSpecString(inferenceElement=InferenceElement.classification,
                               metric=metricName,
                               params={'window':metricWindow},
                               returnLabel=True)

    metricSpecStrings.append(optimizeMetricSpec)

    if options["runBaselines"]:
      # If temporal, generate the trivial predictor metric
      if inferenceType == InferenceType.TemporalClassification:
        metricSpecStrings.append(
          _generateMetricSpecString(inferenceElement=InferenceElement.classification,
                                    metric="trivial",
                                    params={'window':metricWindow,
                                                  "errorMetric":trivialErrorMetric})
          )
        metricSpecStrings.append(
          _generateMetricSpecString(inferenceElement=InferenceElement.classification,
                                    metric="two_gram",
                                    params={'window':metricWindow,
                                                  "errorMetric":oneGramErrorMetric})
          )
        metricSpecStrings.append(
          _generateMetricSpecString(inferenceElement=InferenceElement.classification,
                                    metric=movingAverageBaselineName,
                                    params={'window':metricWindow
                                                  ,"errorMetric":"avg_err",
                                                  "mode_window":200})
          )

    # Custom Error Metric
    if not options["customErrorMetric"] == None :
      #If errorWindow is not specified, make it equal to the default window
      if not "errorWindow" in options["customErrorMetric"]:
        options["customErrorMetric"]["errorWindow"] = metricWindow
      optimizeMetricSpec = _generateMetricSpecString(
                                inferenceElement=InferenceElement.classification,
                                metric="custom",
                                params=options["customErrorMetric"])
      optimizeMetricLabel = ".*custom_error_metric.*"

      metricSpecStrings.append(optimizeMetricSpec)

  # -----------------------------------------------------------------------
  # If plug in the predictionSteps variable for any dynamically generated
  #  prediction steps
  if options['dynamicPredictionSteps']:
    for i in range(len(metricSpecStrings)):
      metricSpecStrings[i] = metricSpecStrings[i].replace(
          "'$REPLACE_ME'", "predictionSteps")
    optimizeMetricLabel = optimizeMetricLabel.replace(
        "'$REPLACE_ME'", ".*")
  return metricSpecStrings, optimizeMetricLabel

Response:
Generates the metric specs for a given InferenceType
 »


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _generateExtraMetricSpecs(options):
  _metricSpecSchema = {'properties': {}}

  results = []
  for metric in options['metrics']:

    for propertyName in _metricSpecSchema['properties'].keys():
      _getPropertyValue(_metricSpecSchema, propertyName, metric)

    specString, label = _generateMetricSpecString(
                                          field=metric['field'],
                                          metric=metric['metric'],
                                          params=metric['params'],
                                          inferenceElement=\
                                                    metric['inferenceElement'],
                                          returnLabel=True)
    if metric['logged']:
      options['loggedMetrics'].append(label)

    results.append(specString)

  return results

Response: Generates the non - default metrics specified by the expGenerator params


Instruction:
How would you implement a function in Python 3 that
gets the predicted field and its datatype from the options dictionary

Response:
def _getPredictedField(options):
  """ Gets the predicted field and it's datatype from the options dictionary

  Returns: (predictedFieldName, predictedFieldType)
  """
  if not options['inferenceArgs'] or \
      not options['inferenceArgs']['predictedField']:
    return None, None

  predictedField = options['inferenceArgs']['predictedField']
  predictedFieldInfo = None
  includedFields = options['includedFields']

  for info in includedFields:
    if info['fieldName'] == predictedField:
      predictedFieldInfo = info
      break

  if predictedFieldInfo is None:
    raise ValueError(
      "Predicted field '%s' does not exist in included fields." % predictedField
    )
  predictedFieldType = predictedFieldInfo['fieldType']

  return predictedField, predictedFieldType


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _generateInferenceArgs(options, tokenReplacements):
  inferenceType = options['inferenceType']
  optionInferenceArgs = options.get('inferenceArgs', None)
  resultInferenceArgs = {}
  predictedField = _getPredictedField(options)[0]

  if inferenceType in (InferenceType.TemporalNextStep,
                       InferenceType.TemporalAnomaly):
    assert predictedField,  "Inference Type '%s' needs a predictedField "\
                            "specified in the inferenceArgs dictionary"\
                            % inferenceType

  if optionInferenceArgs:
    # If we will be using a dynamically created predictionSteps, plug in that
    #  variable name in place of the constant scalar value
    if options['dynamicPredictionSteps']:
      altOptionInferenceArgs = copy.deepcopy(optionInferenceArgs)
      altOptionInferenceArgs['predictionSteps'] = '$REPLACE_ME'
      resultInferenceArgs = pprint.pformat(altOptionInferenceArgs)
      resultInferenceArgs = resultInferenceArgs.replace("'$REPLACE_ME'",
                                                        '[predictionSteps]')
    else:
      resultInferenceArgs = pprint.pformat(optionInferenceArgs)

  tokenReplacements['\$INFERENCE_ARGS'] = resultInferenceArgs

  tokenReplacements['\$PREDICTION_FIELD'] = predictedField

Response:
Generates the token substitutions related to the predicted field
 and the supplemental arguments for prediction
 Generates the token substitutions related to the predicted field
  and the supplemental arguments for prediction


Instruction:
Can you tell what is the following Python 3 function doing
def expGenerator(args):

  # -----------------------------------------------------------------
  # Parse command line options
  #
  parser = OptionParser()
  parser.set_usage("%prog [options] --description='{json object with args}'\n" + \
                   "%prog [options] --descriptionFromFile='{filename}'\n" + \
                   "%prog [options] --showSchema")

  parser.add_option("--description", dest = "description",
    help = "Tells ExpGenerator to generate an experiment description.py and " \
           "permutations.py file using the given JSON formatted experiment "\
           "description string.")

  parser.add_option("--descriptionFromFile", dest = 'descriptionFromFile',
    help = "Tells ExpGenerator to open the given filename and use it's " \
           "contents as the JSON formatted experiment description.")

  parser.add_option("--claDescriptionTemplateFile",
    dest = 'claDescriptionTemplateFile',
    default = 'claDescriptionTemplate.tpl',
    help = "The file containing the template description file for " \
           " ExpGenerator [default: %default]")

  parser.add_option("--showSchema",
                    action="store_true", dest="showSchema",
                    help="Prints the JSON schemas for the --description arg.")

  parser.add_option("--version", dest = 'version', default='v2',
    help = "Generate the permutations file for this version of hypersearch."
            " Possible choices are 'v1' and 'v2' [default: %default].")

  parser.add_option("--outDir",
                    dest = "outDir", default=None,
                    help = "Where to generate experiment. If not specified, " \
                           "then a temp directory will be created"
                    )
  (options, remainingArgs) = parser.parse_args(args)

  #print("OPTIONS=%s" % (str(options)))

  # -----------------------------------------------------------------
  # Check for unprocessed args
  #
  if len(remainingArgs) > 0:
    raise _InvalidCommandArgException(
      _makeUsageErrorStr("Unexpected command-line args: <%s>" % \
                         (' '.join(remainingArgs),), parser.get_usage()))

  # -----------------------------------------------------------------
  # Check for use of mutually-exclusive options
  #
  activeOptions = filter(lambda x: getattr(options, x) != None,
                         ('description', 'showSchema'))
  if len(activeOptions) > 1:
    raise _InvalidCommandArgException(
      _makeUsageErrorStr(("The specified command options are " + \
                          "mutually-exclusive: %s") % (activeOptions,),
                          parser.get_usage()))

  # -----------------------------------------------------------------
  # Process requests
  #
  if options.showSchema:
    _handleShowSchemaOption()

  elif options.description:
    _handleDescriptionOption(options.description, options.outDir,
           parser.get_usage(), hsVersion=options.version,
           claDescriptionTemplateFile = options.claDescriptionTemplateFile)

  elif options.descriptionFromFile:
    _handleDescriptionFromFileOption(options.descriptionFromFile,
          options.outDir, parser.get_usage(), hsVersion=options.version,
          claDescriptionTemplateFile = options.claDescriptionTemplateFile)

  else:
    raise _InvalidCommandArgException(
      _makeUsageErrorStr("Error in validating command options. No option "
                         "provided:\n", parser.get_usage()))

Response: Generates an experiment description file and generates it with the given arguments.


Instruction:
Create a Python 3 function to
parse a textual datetime format and returns a Python datetime object.

Response:
def parseTimestamp(s):
  """
  Parses a textual datetime format and return a Python datetime object.

  The supported format is: ``yyyy-mm-dd h:m:s.ms``

  The time component is optional.

  - hours are 00..23 (no AM/PM)
  - minutes are 00..59
  - seconds are 00..59
  - micro-seconds are 000000..999999

  :param s: (string) input time text
  :return: (datetime.datetime)
  """
  s = s.strip()
  for pattern in DATETIME_FORMATS:
    try:
      return datetime.datetime.strptime(s, pattern)
    except ValueError:
      pass
  raise ValueError('The provided timestamp %s is malformed. The supported '
                   'formats are: [%s]' % (s, ', '.join(DATETIME_FORMATS)))


Instruction:
Can you generate the documentation for the following Python 3 function
def parseBool(s):
  l = s.lower()
  if l in ("true", "t", "1"):
    return True
  if l in ("false", "f", "0"):
    return False
  raise Exception("Unable to convert string '%s' to a boolean value" % s)

Response: Convert string to boolean


Instruction:
Here you have a function in Python 3, explain what it does
def escape(s):
  if s is None:
    return ''

  assert isinstance(s, basestring), \
        "expected %s but got %s; value=%s" % (basestring, type(s), s)
  s = s.replace('\\', '\\\\')
  s = s.replace('\n', '\\n')
  s = s.replace('\t', '\\t')
  s = s.replace(',', '\t')
  return s

Response:
Escape a string in a string
 archive.


Instruction:
Write a Python 3 script to
parse a string containing only 0 and 1 s and returns a Python list object.

Response:
def parseSdr(s):
  """
  Parses a string containing only 0's and 1's and return a Python list object.

  :param s: (string) string to parse
  :returns: (list) SDR out
  """
  assert isinstance(s, basestring)
  sdr = [int(c) for c in s if c in ("0", "1")]
  if len(sdr) != len(s):
    raise ValueError("The provided string %s is malformed. The string should "
                     "have only 0's and 1's.")

  return sdr


Instruction:
How would you code a function in Python 3 to
parse a string of space - separated numbers returning a Python list.

Response:
def parseStringList(s):
  """
  Parse a string of space-separated numbers, returning a Python list.

  :param s: (string) to parse
  :returns: (list) binary SDR
  """
  assert isinstance(s, basestring)
  return [int(i) for i in s.split()]


Instruction:
Can you write a function in Python 3 where it
translates an index into coordinates using the given coordinate system.

Response:
def coordinatesFromIndex(index, dimensions):
  """
  Translate an index into coordinates, using the given coordinate system.

  Similar to ``numpy.unravel_index``.

  :param index: (int) The index of the point. The coordinates are expressed as a
         single index by using the dimensions as a mixed radix definition. For
         example, in dimensions 42x10, the point [1, 4] is index
         1*420 + 4*10 = 460.

  :param dimensions (list of ints) The coordinate system.

  :returns: (list) of coordinates of length ``len(dimensions)``.
  """
  coordinates = [0] * len(dimensions)

  shifted = index
  for i in xrange(len(dimensions) - 1, 0, -1):
    coordinates[i] = shifted % dimensions[i]
    shifted = shifted / dimensions[i]

  coordinates[0] = shifted

  return coordinates


Instruction:
Given the following Python 3 function, write the documentation
def indexFromCoordinates(coordinates, dimensions):
  index = 0
  for i, dimension in enumerate(dimensions):
    index *= dimension
    index += coordinates[i]

  return index

Response: Translate coordinates into an index using the given coordinate system.


Instruction:
Can you generate the documentation for the following Python 3 function
def neighborhood(centerIndex, radius, dimensions):
  centerPosition = coordinatesFromIndex(centerIndex, dimensions)

  intervals = []
  for i, dimension in enumerate(dimensions):
    left = max(0, centerPosition[i] - radius)
    right = min(dimension - 1, centerPosition[i] + radius)
    intervals.append(xrange(left, right + 1))

  coords = numpy.array(list(itertools.product(*intervals)))
  return numpy.ravel_multi_index(coords.T, dimensions)

Response: Return the points in the neighborhood of a point.


Instruction:
Can you generate the documentation for the following Python 3 function
def encodeIntoArray(self, inputData, output):
    (coordinate, radius) = inputData

    assert isinstance(radius, int), ("Expected integer radius, got: {} ({})"
                                     .format(radius, type(radius)))

    neighbors = self._neighbors(coordinate, radius)
    winners = self._topWCoordinates(neighbors, self.w)

    bitFn = lambda coordinate: self._bitForCoordinate(coordinate, self.n)
    indices = numpy.array([bitFn(w) for w in winners])

    output[:] = 0
    output[indices] = 1

Response: Encodes the SDR in the input numpy array into the output numpy array.


Instruction:
Make a summary of the following Python 3 code
def _neighbors(coordinate, radius):
    ranges = (xrange(n-radius, n+radius+1) for n in coordinate.tolist())
    return numpy.array(list(itertools.product(*ranges)))

Response: Returns coordinates around given coordinate within given radius.


Instruction:
Write a Python 3 function that can
return the top W coordinates by order.

Response:
def _topWCoordinates(cls, coordinates, w):
    """
    Returns the top W coordinates by order.

    @param coordinates (numpy.array) A 2D numpy array, where each element
                                     is a coordinate
    @param w (int) Number of top coordinates to return
    @return (numpy.array) A subset of `coordinates`, containing only the
                          top ones by order
    """
    orders = numpy.array([cls._orderForCoordinate(c)
                          for c in coordinates.tolist()])
    indices = numpy.argsort(orders)[-w:]
    return coordinates[indices]


Instruction:
Create a Python 3 function for
hashing a coordinate to a 64 bit integer.

Response:
def _hashCoordinate(coordinate):
    """Hash a coordinate to a 64 bit integer."""
    coordinateStr = ",".join(str(v) for v in coordinate)
    # Compute the hash and convert to 64 bit int.
    hash = int(int(hashlib.md5(coordinateStr).hexdigest(), 16) % (2 ** 64))
    return hash


Instruction:
How would you code a function in Python 3 to
return the order for a coordinate.

Response:
def _orderForCoordinate(cls, coordinate):
    """
    Returns the order for a coordinate.

    @param coordinate (numpy.array) Coordinate
    @return (float) A value in the interval [0, 1), representing the
                    order of the coordinate
    """
    seed = cls._hashCoordinate(coordinate)
    rng = Random(seed)
    return rng.getReal64()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _bitForCoordinate(cls, coordinate, n):
    seed = cls._hashCoordinate(coordinate)
    rng = Random(seed)
    return rng.getUInt32(n)

Response: Maps the coordinate to a bit in the SDR.


Instruction:
Can you tell what is the following Python 3 function doing
def binSearch(arr, val):
  i = bisect_left(arr, val)
  if i != len(arr) and arr[i] == val:
    return i
  return -1

Response: Function for running binary search on a sorted list.


Instruction:
Write a Python 3 script for
adding a new segment on a cell.

Response:
def createSegment(self, cell):
    """
    Adds a new segment on a cell.

    :param cell: (int) Cell index
    :returns: (int) New segment index
    """
    cellData = self._cells[cell]

    if len(self._freeFlatIdxs) > 0:
      flatIdx = self._freeFlatIdxs.pop()
    else:
      flatIdx = self._nextFlatIdx
      self._segmentForFlatIdx.append(None)
      self._nextFlatIdx += 1

    ordinal = self._nextSegmentOrdinal
    self._nextSegmentOrdinal += 1

    segment = Segment(cell, flatIdx, ordinal)
    cellData._segments.append(segment)
    self._segmentForFlatIdx[flatIdx] = segment

    return segment


Instruction:
Implement a function in Python 3 to
destroy a segment. :param segment: (:class:`Segment`) representing the segment to be destroyed.

Response:
def destroySegment(self, segment):
    """
    Destroys a segment.

    :param segment: (:class:`Segment`) representing the segment to be destroyed.
    """
    # Remove the synapses from all data structures outside this Segment.
    for synapse in segment._synapses:
      self._removeSynapseFromPresynapticMap(synapse)
    self._numSynapses -= len(segment._synapses)

    # Remove the segment from the cell's list.
    segments = self._cells[segment.cell]._segments
    i = segments.index(segment)
    del segments[i]

    # Free the flatIdx and remove the final reference so the Segment can be
    # garbage-collected.
    self._freeFlatIdxs.append(segment.flatIdx)
    self._segmentForFlatIdx[segment.flatIdx] = None


Instruction:
Implement a function in Python 3 to
create a new synapse on a segment.

Response:
def createSynapse(self, segment, presynapticCell, permanence):
    """
    Creates a new synapse on a segment.

    :param segment: (:class:`Segment`) Segment object for synapse to be synapsed
           to.
    :param presynapticCell: (int) Source cell index.
    :param permanence: (float) Initial permanence of synapse.
    :returns: (:class:`Synapse`) created synapse
    """
    idx = len(segment._synapses)
    synapse = Synapse(segment, presynapticCell, permanence,
                      self._nextSynapseOrdinal)
    self._nextSynapseOrdinal += 1
    segment._synapses.add(synapse)

    self._synapsesForPresynapticCell[presynapticCell].add(synapse)

    self._numSynapses += 1

    return synapse


Instruction:
Given the following Python 3 function, write the documentation
def destroySynapse(self, synapse):

    self._numSynapses -= 1

    self._removeSynapseFromPresynapticMap(synapse)

    synapse.segment._synapses.remove(synapse)

Response:
Destroys a synapse.

    :param synapse: (:class:`Synapse`) synapse to destroy


Instruction:
Explain what the following Python 3 code does
def computeActivity(self, activePresynapticCells, connectedPermanence):

    numActiveConnectedSynapsesForSegment = [0] * self._nextFlatIdx
    numActivePotentialSynapsesForSegment = [0] * self._nextFlatIdx

    threshold = connectedPermanence - EPSILON

    for cell in activePresynapticCells:
      for synapse in self._synapsesForPresynapticCell[cell]:
        flatIdx = synapse.segment.flatIdx
        numActivePotentialSynapsesForSegment[flatIdx] += 1
        if synapse.permanence > threshold:
          numActiveConnectedSynapsesForSegment[flatIdx] += 1

    return (numActiveConnectedSynapsesForSegment,
            numActivePotentialSynapsesForSegment)

Response: Compute each segment s number of active synapses for a given input.


Instruction:
Make a summary of the following Python 3 code
def numSegments(self, cell=None):
    if cell is not None:
      return len(self._cells[cell]._segments)

    return self._nextFlatIdx - len(self._freeFlatIdxs)

Response: Returns the number of segments on a specific cell.


Instruction:
Can you tell what is the following Python 3 function doing
def segmentPositionSortKey(self, segment):
    return segment.cell + (segment._ordinal / float(self._nextSegmentOrdinal))

Response:
Returns a numeric key for sorting this segment. This can be used with the
    python built - in sorted function.


Instruction:
Explain what the following Python 3 code does
def write(self, proto):
    protoCells = proto.init('cells', self.numCells)

    for i in xrange(self.numCells):
      segments = self._cells[i]._segments
      protoSegments = protoCells[i].init('segments', len(segments))

      for j, segment in enumerate(segments):
        synapses = segment._synapses
        protoSynapses = protoSegments[j].init('synapses', len(synapses))

        for k, synapse in enumerate(sorted(synapses, key=lambda s: s._ordinal)):
          protoSynapses[k].presynapticCell = synapse.presynapticCell
          protoSynapses[k].permanence = synapse.permanence

Response: Writes serialized data to proto object.


Instruction:
How would you implement a function in Python 3 that
reads deserialized data from proto object

Response:
def read(cls, proto):
    """
    Reads deserialized data from proto object

    :param proto: (DynamicStructBuilder) Proto object

    :returns: (:class:`Connections`) instance
    """
    #pylint: disable=W0212
    protoCells = proto.cells
    connections = cls(len(protoCells))

    for cellIdx, protoCell in enumerate(protoCells):
      protoCell = protoCells[cellIdx]
      protoSegments = protoCell.segments
      connections._cells[cellIdx] = CellData()
      segments = connections._cells[cellIdx]._segments

      for segmentIdx, protoSegment in enumerate(protoSegments):
        segment = Segment(cellIdx, connections._nextFlatIdx,
                          connections._nextSegmentOrdinal)

        segments.append(segment)
        connections._segmentForFlatIdx.append(segment)
        connections._nextFlatIdx += 1
        connections._nextSegmentOrdinal += 1

        synapses = segment._synapses
        protoSynapses = protoSegment.synapses

        for synapseIdx, protoSynapse in enumerate(protoSynapses):
          presynapticCell = protoSynapse.presynapticCell
          synapse = Synapse(segment, presynapticCell, protoSynapse.permanence,
                            ordinal=connections._nextSynapseOrdinal)
          connections._nextSynapseOrdinal += 1
          synapses.add(synapse)
          connections._synapsesForPresynapticCell[presynapticCell].add(synapse)

          connections._numSynapses += 1

    #pylint: enable=W0212
    return connections


Instruction:
How would you code a function in Python 3 to
retrieve the requested property as a string.

Response:
def getString(cls, prop):
    """ Retrieve the requested property as a string. If property does not exist,
    then KeyError will be raised.

    :param prop: (string) name of the property
    :raises: KeyError
    :returns: (string) property value
    """
    if cls._properties is None:
      cls._readStdConfigFiles()

    # Allow configuration properties to be overridden via environment variables
    envValue = os.environ.get("%s%s" % (cls.envPropPrefix,
                                        prop.replace('.', '_')), None)
    if envValue is not None:
      return envValue

    return cls._properties[prop]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def getBool(cls, prop):

    value = cls.getInt(prop)

    if value not in (0, 1):
      raise ValueError("Expected 0 or 1, but got %r in config property %s" % (
        value, prop))

    return bool(value)

Response: Retrieve the requested property and return it as a bool.


Instruction:
Can you generate the documentation for the following Python 3 function
def set(cls, prop, value):

    if cls._properties is None:
      cls._readStdConfigFiles()

    cls._properties[prop] = str(value)

Response: Set the value of the given configuration property.


Instruction:
How would you explain what the following Python 3 function does
def dict(cls):

    if cls._properties is None:
      cls._readStdConfigFiles()

    # Make a copy so we can update any current values obtained from environment
    #  variables
    result = dict(cls._properties)
    keys = os.environ.keys()
    replaceKeys = filter(lambda x: x.startswith(cls.envPropPrefix),
                         keys)
    for envKey in replaceKeys:
      key = envKey[len(cls.envPropPrefix):]
      key = key.replace('_', '.')
      result[key] = os.environ[envKey]

    return result

Response: Returns a dict containing all of the configuration properties containing all of the configuration properties in the system.


Instruction:
Can you generate the documentation for the following Python 3 function
def readConfigFile(cls, filename, path=None):
    properties = cls._readConfigFile(filename, path)

    # Create properties dict if necessary
    if cls._properties is None:
      cls._properties = dict()

    for name in properties:
      if 'value' in properties[name]:
        cls._properties[name] = properties[name]['value']

Response: Parse the given XML file and store all properties it describes.


Instruction:
Implement a function in Python 3 to
search the configuration path for the given filename.

Response:
def findConfigFile(cls, filename):
    """ Search the configuration path (specified via the NTA_CONF_PATH
    environment variable) for the given filename. If found, return the complete
    path to the file.

    :param filename: (string) name of file to locate
    """

    paths = cls.getConfigPaths()
    for p in paths:
      testPath = os.path.join(p, filename)
      if os.path.isfile(testPath):
        return os.path.join(p, filename)


Instruction:
Can you generate the documentation for the following Python 3 function
def getConfigPaths(cls):
    configPaths = []
    if cls._configPaths is not None:
      return cls._configPaths

    else:
      if 'NTA_CONF_PATH' in os.environ:
        configVar = os.environ['NTA_CONF_PATH']
        # Return as a list of paths
        configPaths = configVar.split(os.pathsep)

      return configPaths

Response: Return the list of paths to search for configuration files.


Instruction:
Can you create a Python 3 function that
adds noise to the input.

Response:
def addNoise(input, noise=0.1, doForeground=True, doBackground=True):
  """
  Add noise to the given input.

  Parameters:
  -----------------------------------------------
  input:         the input to add noise to
  noise:         how much noise to add
  doForeground:  If true, turn off some of the 1 bits in the input
  doBackground:  If true, turn on some of the 0 bits in the input

  """
  if doForeground and doBackground:
    return numpy.abs(input -  (numpy.random.random(input.shape) < noise))
  else:
    if doForeground:
      return numpy.logical_and(input, numpy.random.random(input.shape) > noise)
    if doBackground:
      return numpy.logical_or(input, numpy.random.random(input.shape) < noise)
  return input


Instruction:
Can you create a Python 3 function that
generates a coincidence matrix.

Response:
def generateCoincMatrix(nCoinc=10, length=500, activity=50):
  """
  Generate a coincidence matrix. This is used to generate random inputs to the
  temporal learner and to compare the predicted output against.

  It generates a matrix of nCoinc rows, each row has length 'length' and has
  a total of 'activity' bits on.

  Parameters:
  -----------------------------------------------
  nCoinc:        the number of rows to generate
  length:        the length of each row
  activity:      the number of ones to put into each row.

  """

  coincMatrix0 = SM32(int(nCoinc), int(length))
  theOnes = numpy.array([1.0] * activity, dtype=numpy.float32)
  for rowIdx in xrange(nCoinc):
    coinc = numpy.array(random.sample(xrange(length),
                activity), dtype=numpy.uint32)
    coinc.sort()
    coincMatrix0.setRowFromSparse(rowIdx, coinc, theOnes)

  # This is the right code to use, it's faster, but it derails the unit
  # testing of the pooling for now.
  coincMatrix = SM32(int(nCoinc), int(length))
  coincMatrix.initializeWithFixedNNZR(activity)

  return coincMatrix0


Instruction:
How would you code a function in Python 3 to
generate a list of random sparse distributed vectors.

Response:
def generateVectors(numVectors=100, length=500, activity=50):
  """
  Generate a list of random sparse distributed vectors.  This is used to generate
  training vectors to the spatial or temporal learner and to compare the predicted
  output against.

  It generates a list of 'numVectors' elements, each element has length 'length'
  and has a total of 'activity' bits on.

  Parameters:
  -----------------------------------------------
  numVectors:    the number of vectors to generate
  length:        the length of each row
  activity:      the number of ones to put into each row.

  """

  vectors = []
  coinc = numpy.zeros(length, dtype='int32')
  indexList = range(length)

  for i in xrange(numVectors):
      coinc[:] = 0
      coinc[random.sample(indexList, activity)] = 1
      vectors.append(coinc.copy())

  return vectors


Instruction:
Can you tell what is the following Python 3 function doing
def generateSimpleSequences(nCoinc=10, seqLength=[5,6,7], nSeq=100):

  coincList = range(nCoinc)
  seqList  = []

  for i in xrange(nSeq):
    if max(seqLength) <= nCoinc:
      seqList.append(random.sample(coincList, random.choice(seqLength)))
    else:
      len = random.choice(seqLength)
      seq = []
      for x in xrange(len):
        seq.append(random.choice(coincList))
      seqList.append(seq)

  return seqList

Response: Generates a set of simple sequences.


Instruction:
Here you have a function in Python 3, explain what it does
def generateHubSequences(nCoinc=10, hubs = [2,6], seqLength=[5,6,7], nSeq=100):

  coincList = range(nCoinc)
  for hub in hubs:
    coincList.remove(hub)

  seqList = []
  for i in xrange(nSeq):
    length = random.choice(seqLength)-1
    seq = random.sample(coincList,length)
    seq.insert(length//2, random.choice(hubs))
    seqList.append(seq)

  return seqList

Response: Generates a list of hubs that are available in the middle of the hubs.


Instruction:
Create a Python 3 function for
generating a non overlapping coincidence matrix.

Response:
def generateSimpleCoincMatrix(nCoinc=10, length=500, activity=50):
  """
  Generate a non overlapping coincidence matrix. This is used to generate random
  inputs to the temporal learner and to compare the predicted output against.

  It generates a matrix of nCoinc rows, each row has length 'length' and has
  a total of 'activity' bits on.

  Parameters:
  -----------------------------------------------
  nCoinc:        the number of rows to generate
  length:        the length of each row
  activity:      the number of ones to put into each row.

  """
  assert nCoinc*activity<=length, "can't generate non-overlapping coincidences"
  coincMatrix = SM32(0, length)
  coinc = numpy.zeros(length, dtype='int32')

  for i in xrange(nCoinc):
      coinc[:] = 0
      coinc[i*activity:(i+1)*activity] = 1
      coincMatrix.addRow(coinc)

  return coincMatrix


Instruction:
Given the following Python 3 function, write the documentation
def generateSequences(nPatterns=10, patternLen=500, patternActivity=50,
                    hubs=[2,6],  seqLength=[5,6,7],
                    nSimpleSequences=50,  nHubSequences=50):
  """
  Generate a set of simple and hub sequences. A simple sequence contains
  a randomly chosen set of elements from 0 to 'nCoinc-1'. A hub sequence
  always contains a hub element in the middle of it.

  Parameters:
  -----------------------------------------------
  nPatterns:        the number of patterns to use in the sequences.
  patternLen:       The number of elements in each pattern
  patternActivity:  The number of elements that should be active in
                        each pattern
  hubs:             which of the elements will be used as hubs.
  seqLength:        a list of possible sequence lengths. The length of each
                        sequence will be randomly chosen from here.
  nSimpleSequences: The number of simple sequences to generate
  nHubSequences:    The number of hub sequences to generate

  retval:           (seqList, patterns)
                    seqList: a list of sequences. Each sequence is itself a list
                                  containing the input pattern indices for that sequence.
                    patterns: the input patterns used in the seqList.
  """

  # Create the input patterns
  patterns = generateCoincMatrix(nCoinc=nPatterns, length=patternLen,
              activity=patternActivity)

  # Create the raw sequences
  seqList =  generateSimpleSequences(nCoinc=nPatterns, seqLength=seqLength,
                                    nSeq=nSimpleSequences) + \
             generateHubSequences(nCoinc=nPatterns, hubs=hubs, seqLength=seqLength,
                                  nSeq=nHubSequences)

  # Return results
  return (seqList, patterns)

Response: Generates a set of simple and hub sequences.


Instruction:
Write a Python 3 function for
generating the simulated output from a spatial pooler that's sitting on top of another spatial pooler / temporal memory pair. The average on-time of the outputs from the simulated TM is given by the l1Pooling argument. In this routine, L1 refers to the first spatial and temporal memory and L2 refers to the spatial pooler above that. Parameters: ----------------------------------------------- nL1Patterns: the number of patterns to use in the L1 sequences. l1Hubs: which of the elements will be used as hubs. l1SeqLength: a list of possible sequence lengths. The length of each sequence will be randomly chosen from here. nL1SimpleSequences: The number of simple sequences to generate for L1 nL1HubSequences: The number of hub sequences to generate for L1 l1Pooling: The number of time steps to pool over in the L1 temporal pooler perfectStability: If true, then the input patterns represented by the sequences generated will have perfect stability over l1Pooling time steps. This is the best case ideal input to a TM. In actual situations, with an actual SP providing input, the stability will always be less than this. spHystereisFactor: The hysteresisFactor to use in the L2 spatial pooler. Only used when perfectStability is False patternLen: The number of elements in each pattern output by L2 patternActivity: The number of elements that should be active in each pattern @retval: (seqList, patterns) seqList: a list of sequences output from L2. Each sequence is itself a list containing the input pattern indices for that sequence. patterns: the input patterns used in the L2 seqList.

Response:
def generateL2Sequences(nL1Patterns=10, l1Hubs=[2,6], l1SeqLength=[5,6,7],
                  nL1SimpleSequences=50,  nL1HubSequences=50,
                  l1Pooling=4, perfectStability=False, spHysteresisFactor=1.0,
                  patternLen=500, patternActivity=50):
  """
  Generate the simulated output from a spatial pooler that's sitting
  on top of another spatial pooler / temporal memory pair.  The average on-time
  of the outputs from the simulated TM is given by the l1Pooling argument.

  In this routine, L1 refers to the first spatial and temporal memory and L2
  refers to the spatial pooler above that.

  Parameters:
  -----------------------------------------------
  nL1Patterns:          the number of patterns to use in the L1 sequences.
  l1Hubs:               which of the elements will be used as hubs.
  l1SeqLength:          a list of possible sequence lengths. The length of each
                        sequence will be randomly chosen from here.
  nL1SimpleSequences:   The number of simple sequences to generate for L1
  nL1HubSequences:      The number of hub sequences to generate for L1
  l1Pooling:            The number of time steps to pool over in the L1 temporal
                          pooler
  perfectStability:     If true, then the input patterns represented by the
                        sequences generated will have perfect stability over
                        l1Pooling time steps. This is the best case ideal input
                        to a TM. In actual situations, with an actual SP
                        providing input, the stability will always be less than
                        this.
  spHystereisFactor:    The hysteresisFactor to use in the L2 spatial pooler.
                        Only used when perfectStability is  False
  patternLen:           The number of elements in each pattern output by L2
  patternActivity:      The number of elements that should be active in
                        each pattern

  @retval:              (seqList, patterns)
                        seqList: a list of sequences output from L2. Each sequence is
                            itself a list containing the input pattern indices for that
                            sequence.
                        patterns: the input patterns used in the L2 seqList.
  """

  # First, generate the L1 sequences
  l1SeqList = generateSimpleSequences(nCoinc=nL1Patterns, seqLength=l1SeqLength,
                                    nSeq=nL1SimpleSequences) + \
             generateHubSequences(nCoinc=nL1Patterns, hubs=l1Hubs,
                                    seqLength=l1SeqLength, nSeq=nL1HubSequences)

  # Generate the L2 SP output from those
  spOutput = generateSlowSPOutput(seqListBelow = l1SeqList,
                poolingTimeBelow=l1Pooling, outputWidth=patternLen,
                activity=patternActivity, perfectStability=perfectStability,
                spHysteresisFactor=spHysteresisFactor)

  # Map the spOutput patterns into indices into a pattern matrix which we
  #  generate now.
  outSeq = None
  outSeqList = []
  outPatterns = SM32(0, patternLen)
  for pattern in spOutput:
    # If we have a reset vector start a new sequence
    if pattern.sum() == 0:
      if outSeq is not None:
        outSeqList.append(outSeq)
      outSeq = []
      continue

    # See if this vector matches a pattern we've already seen before
    patternIdx = None
    if outPatterns.nRows() > 0:
      # Find most matching 1's.
      matches = outPatterns.rightVecSumAtNZ(pattern)
      outCoinc = matches.argmax().astype('uint32')
      # See if its number of 1's is the same in the pattern and in the
      #  coincidence row. If so, it is an exact match
      numOnes = pattern.sum()
      if matches[outCoinc] == numOnes \
          and outPatterns.getRow(int(outCoinc)).sum() == numOnes:
        patternIdx = outCoinc

    # If no match, add this pattern to our matrix
    if patternIdx is None:
      outPatterns.addRow(pattern)
      patternIdx = outPatterns.nRows() - 1

    # Store the pattern index into the sequence
    outSeq.append(patternIdx)

  # Put in last finished sequence
  if outSeq is not None:
    outSeqList.append(outSeq)

  # Return with the seqList and patterns matrix
  return (outSeqList, outPatterns)


Instruction:
Can you generate the documentation for the following Python 3 function
def vectorsFromSeqList(seqList, patternMatrix):

  totalLen = 0
  for seq in seqList:
    totalLen += len(seq)

  vectors = numpy.zeros((totalLen, patternMatrix.shape[1]), dtype='bool')
  vecOffset = 0
  for seq in seqList:
    seq = numpy.array(seq, dtype='uint32')
    for idx,coinc in enumerate(seq):
      vectors[vecOffset] = patternMatrix.getRow(int(coinc))
      vecOffset += 1

  return vectors

Response: Returns a list of vectors from a list of sequences of pattern indices and a pattern lookup table


Instruction:
Here you have a function in Python 3, explain what it does
def sameTMParams(tp1, tp2):
  result = True
  for param in ["numberOfCols", "cellsPerColumn", "initialPerm", "connectedPerm",
                "minThreshold", "newSynapseCount", "permanenceInc", "permanenceDec",
                "permanenceMax", "globalDecay", "activationThreshold",
                "doPooling", "segUpdateValidDuration",
                "burnIn", "pamLength", "maxAge"]:
    if getattr(tp1, param) != getattr(tp2,param):
      print param,"is different"
      print getattr(tp1, param), "vs", getattr(tp2,param)
      result = False
  return result

Response: Given two TM instances see if any parameters are different.


Instruction:
How would you explain what the following Python 3 function does
def sameSynapse(syn, synapses):
  for s in synapses:
    if (s[0]==syn[0]) and (s[1]==syn[1]) and (abs(s[2]-syn[2]) <= 0.001):
      return True
  return False

Response: Checks whether this synapse and a list of synapses exist in the list.


Instruction:
Can you write a function in Python 3 where it
returns True if seg1 and seg2 are identical ignoring order of synapses

Response:
def sameSegment(seg1, seg2):
  """Return True if seg1 and seg2 are identical, ignoring order of synapses"""
  result = True

  # check sequence segment, total activations etc. In case any are floats,
  # check that they are within 0.001.
  for field in [1, 2, 3, 4, 5, 6]:
    if abs(seg1[0][field] - seg2[0][field]) > 0.001:
      result = False

  # Compare number of synapses
  if len(seg1[1:]) != len(seg2[1:]):
    result = False

  # Now compare synapses, ignoring order of synapses
  for syn in seg2[1:]:
    if syn[2] <= 0:
      print "A synapse with zero permanence encountered"
      result = False
  if result == True:
    for syn in seg1[1:]:
      if syn[2] <= 0:
        print "A synapse with zero permanence encountered"
        result = False
      res = sameSynapse(syn, seg2[1:])
      if res == False:
        result = False

  return result


Instruction:
Here you have a function in Python 3, explain what it does
def tmDiff(tm1, tm2, verbosity = 0, relaxSegmentTests =True):

  # First check basic parameters. If we fail here, don't continue
  if sameTMParams(tm1, tm2) == False:
    print "Two TM's have different parameters"
    return False

  result = True

  # Compare states at t first, they usually diverge before the structure of the
  # cells starts diverging

  if (tm1.activeState['t'] != tm2.activeState['t']).any():
    print 'Active states diverge', numpy.where(tm1.activeState['t'] != tm2.activeState['t'])
    result = False

  if (tm1.predictedState['t'] - tm2.predictedState['t']).any():
    print 'Predicted states diverge', numpy.where(tm1.predictedState['t'] != tm2.predictedState['t'])
    result = False

  # TODO: check confidence at T (confT)

  # Now check some high level learned parameters.
  if tm1.getNumSegments() != tm2.getNumSegments():
    print "Number of segments are different", tm1.getNumSegments(), tm2.getNumSegments()
    result = False

  if tm1.getNumSynapses() != tm2.getNumSynapses():
    print "Number of synapses are different", tm1.getNumSynapses(), tm2.getNumSynapses()
    tm1.printCells()
    tm2.printCells()
    result = False

  # Check that each cell has the same number of segments and synapses
  for c in xrange(tm1.numberOfCols):
    for i in xrange(tm2.cellsPerColumn):
      if tm1.getNumSegmentsInCell(c, i) != tm2.getNumSegmentsInCell(c, i):
        print "Num segments different in cell:",c,i,
        print tm1.getNumSegmentsInCell(c, i), tm2.getNumSegmentsInCell(c, i)
        result = False

  # If the above tests pass, then check each segment and report differences
  # Note that segments in tm1 can be in a different order than tm2. Here we
  # make sure that, for each segment in tm1, there is an identical segment
  # in tm2.
  if result == True and not relaxSegmentTests:
    for c in xrange(tm1.numberOfCols):
      for i in xrange(tm2.cellsPerColumn):
        nSegs = tm1.getNumSegmentsInCell(c, i)
        for segIdx in xrange(nSegs):
          tm1seg = tm1.getSegmentOnCell(c, i, segIdx)

          # Loop through all segments in tm2seg and see if any of them match tm1seg
          res = False
          for tm2segIdx in xrange(nSegs):
            tm2seg = tm2.getSegmentOnCell(c, i, tm2segIdx)
            if sameSegment(tm1seg, tm2seg) == True:
              res = True
              break
          if res == False:
            print "\nSegments are different for cell:",c,i
            if verbosity >= 1:
              print "C++"
              tm1.printCell(c, i)
              print "Py"
              tm2.printCell(c, i)
            result = False

  if result == True and (verbosity > 1):
    print "TM's match"

  return result

Response: This function compares two TM instances and returns a list of difference between them.


Instruction:
Implement a function in Python 3 to
return True if tm1 - tm2 is a difference between tm1 and tm2.

Response:
def tmDiff2(tm1, tm2, verbosity = 0, relaxSegmentTests =True,
            checkLearn = True, checkStates = True):
  """
  Given two TM instances, list the difference between them and returns False
  if there is a difference. This function checks the major parameters. If this
  passes (and checkLearn is true) it checks the number of segments on each cell.
  If this passes, checks each synapse on each segment.
  When comparing C++ and Py, the segments are usually in different orders in the
  cells. tmDiff ignores segment order when comparing TM's.

  If checkLearn is True, will check learn states as well as all the segments

  If checkStates is True, will check the various state arrays

  """

  # First check basic parameters. If we fail here, don't continue
  if sameTMParams(tm1, tm2) == False:
    print "Two TM's have different parameters"
    return False

  tm1Label = "<tm_1 (%s)>" % tm1.__class__.__name__
  tm2Label = "<tm_2 (%s)>" % tm2.__class__.__name__

  result = True

  if checkStates:
    # Compare states at t first, they usually diverge before the structure of the
    # cells starts diverging

    if (tm1.infActiveState['t'] != tm2.infActiveState['t']).any():
      print 'Active states diverged', numpy.where(tm1.infActiveState['t'] != tm2.infActiveState['t'])
      result = False

    if (tm1.infPredictedState['t'] - tm2.infPredictedState['t']).any():
      print 'Predicted states diverged', numpy.where(tm1.infPredictedState['t'] != tm2.infPredictedState['t'])
      result = False

    if checkLearn and (tm1.lrnActiveState['t'] - tm2.lrnActiveState['t']).any():
      print 'lrnActiveState[t] diverged', numpy.where(tm1.lrnActiveState['t'] != tm2.lrnActiveState['t'])
      result = False

    if checkLearn and (tm1.lrnPredictedState['t'] - tm2.lrnPredictedState['t']).any():
      print 'lrnPredictedState[t] diverged', numpy.where(tm1.lrnPredictedState['t'] != tm2.lrnPredictedState['t'])
      result = False

    if checkLearn and abs(tm1.getAvgLearnedSeqLength() - tm2.getAvgLearnedSeqLength()) > 0.01:
      print "Average learned sequence lengths differ: ",
      print tm1.getAvgLearnedSeqLength(), " vs ", tm2.getAvgLearnedSeqLength()
      result = False

  # TODO: check confidence at T (confT)

  # Now check some high level learned parameters.
  if tm1.getNumSegments() != tm2.getNumSegments():
    print "Number of segments are different", tm1.getNumSegments(), tm2.getNumSegments()
    result = False

  if tm1.getNumSynapses() != tm2.getNumSynapses():
    print "Number of synapses are different", tm1.getNumSynapses(), tm2.getNumSynapses()
    if verbosity >= 3:
      print "%s: " % tm1Label,
      tm1.printCells()
      print "\n%s  : " % tm2Label,
      tm2.printCells()
    #result = False

  # Check that each cell has the same number of segments and synapses
  for c in xrange(tm1.numberOfCols):
    for i in xrange(tm2.cellsPerColumn):
      if tm1.getNumSegmentsInCell(c, i) != tm2.getNumSegmentsInCell(c, i):
        print "Num segments different in cell:",c,i,
        print tm1.getNumSegmentsInCell(c, i), tm2.getNumSegmentsInCell(c, i)
        result = False

  # If the above tests pass, then check each segment and report differences
  # Note that segments in tm1 can be in a different order than tm2. Here we
  # make sure that, for each segment in tm1, there is an identical segment
  # in tm2.
  if result == True and not relaxSegmentTests and checkLearn:
    for c in xrange(tm1.numberOfCols):
      for i in xrange(tm2.cellsPerColumn):
        nSegs = tm1.getNumSegmentsInCell(c, i)
        for segIdx in xrange(nSegs):
          tm1seg = tm1.getSegmentOnCell(c, i, segIdx)

          # Loop through all segments in tm2seg and see if any of them match tm1seg
          res = False
          for tm2segIdx in xrange(nSegs):
            tm2seg = tm2.getSegmentOnCell(c, i, tm2segIdx)
            if sameSegment(tm1seg, tm2seg) == True:
              res = True
              break
          if res == False:
            print "\nSegments are different for cell:",c,i
            result = False
            if verbosity >= 0:
              print "%s : " % tm1Label,
              tm1.printCell(c, i)
              print "\n%s  : " % tm2Label,
              tm2.printCell(c, i)

  if result == True and (verbosity > 1):
    print "TM's match"

  return result