"""
Copyright (c) 2018 Mejbah ul Alam, Justin Gottschlich, Abdullah Muzahid

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""

"""
File name: autoperf.py
File description: Main script for training models and test AutoPerf
"""

import sys
import numpy as np
import matplotlib as mpl
mpl.use('Agg') #instead of Xserver for png
import matplotlib.pyplot as plt
import pprint
import numpy as np
import plot_utils
import os
import errno
import configs
from utils import *
from random import shuffle
import keras_autoencoder
import copy
from collections import namedtuple

"""
Defintions of tuples used by this script
"""
AnomalyTuple = namedtuple('AnomalyTuple', 'run, sample_count, anomalous_sample_count, ranking')
AccuracyTuple = namedtuple('AccuracyTuple', 'true_positive, false_negative, false_positive, true_negative')

"""
Create list of sample with all the counter values from profile data
"""
def getPerfDataset( dirName , numberOfCounters ):
  datasetHeader = []
  dataset = []
  eventID = 0
  for i in range(0, numberOfCounters):
    #if i==2 or i==15 or  i==16  :
    #  continue #TODO: 2 counters are not set in PAPI, temp fix , remove this once problem is resolved

    filename = dirName + "/event_" + str(eventID) + "_perf_data.csv"
    eventID += 1
    with open(filename, 'r') as fp:
      for linenumber,line in enumerate(fp):
        if linenumber == 2:
          headers = line.strip().split(",")   #last one is the counter, 1 and 2  is thd id and instcouunt , 0 is mark id
          datasetHeader.append(headers[-1])
        if linenumber > 2:
          perfCounters = line.strip().split(",")
          mark = int(perfCounters[0])
          threadCount = int(perfCounters[1])
          instructionCount = int(perfCounters[2])
          currCounter = int(perfCounters[3])

          normalizedCounter = ( currCounter / ( instructionCount * threadCount ) )* configs.SCALE_UP_FACTOR
          if i==0:
            newSample = []
            newSample.append(normalizedCounter)
            dataset.append(newSample)
            dataset[linenumber-3].append(normalizedCounter)

  return datasetHeader, dataset

"""
Convert list to numpy array
"""
def getDatasetArray( dataset ):

  dataArray = np.array(dataset, dtype='float32')
  return dataArray

"""
Error function definition
"""
def getMSE(old, new):
  squaredError = ((old - new) ** 2)
  return np.mean(squaredError)
  #return mean_squared_error(old,new)

"""
Distance calculation between reconstructed and original inputs
"""
def getNormalizedDistance( old, new ):
  dist = np.linalg.norm(old-new)
  origin = np.linalg.norm(old)
  return dist/origin

"""
Function for ranking of anomalies
in : list of tuple of (event,error)
out : ranking and majority voting update
"""
def rankAnomalousPoint( sampleErrorsTuple, rankingMap ):

  sampleErrorsTuple.sort(key=lambda tup : tup[1], reverse=True)
  for index, errorTuple in enumerate(sampleErrorsTuple):
	  rankingMap[errorTuple[0]][index] += 1

  return rankingMap

"""
Report ranking based on majority voting result
in : rankingMap -- key: counter_name, val: vote count for each pos/rank
out: list of rank for the entire execution anomaly root cause
"""
def reportRanks( rankingMap ) :

  anomalousCounterRank = []
  for i in range(configs.NUMBER_OF_COUNTERS): ##for each possible ranks
    maxVote = 0
    maxkey = "None"
    for key in rankingMap:
      if rankingMap[key][i] > maxVote:
        maxVote = rankingMap[key][i]
        maxkey = key

    anomalousCounterRank.append(maxkey)

  return anomalousCounterRank

"""
Calculate reconstruction error : normalized distance
"""
def detectAnomalyPoints( realData, predictedData, datasetHeader, thresholdLoss, outFile=None ):
  datasetLen = realData.shape[0]
  dataLen = realData.shape[1]
  anomalyCount = 0

  """
  map for ranking :
  key = counter name, val = list of len(NUMBER_OF_COUNTERS)
  each pos corresponds to rank, smaller is better
  """
  rankingMap = {}
  for counterName in datasetHeader:
    rankingMap[counterName] = [0] * configs.NUMBER_OF_COUNTERS

  reconstructErrorList = []
  for x  in range(datasetLen):
    #reconstructionError = getNormalizedDistance( realData[x], predictedData[x] )
    reconstructionError = getMSE( realData[x], predictedData[x] )
    #for debugging
    reconstructErrorList.append(reconstructionError)
    #end for debugging
    #if(reconstructionError > configs.THRESHOLD_ERROR):
    if(reconstructionError > thresholdLoss):
      anomalyCount += 1
      errorList = [] #for ranking
      for y in range(0, dataLen):
        dist=abs(predictedData[x][y]-realData[x][y]) / realData[x][y]
        #collect errors and counter tuple
        errorList.append( (datasetHeader[y], dist) )
      #update ranking
      rankingMap = rankAnomalousPoint(errorList, rankingMap)

  votingResult = reportRanks( rankingMap )

  return datasetLen, anomalyCount, votingResult

"""
Calculate a threshold based on distribution of reconstrucion errors
"""
def getReconstructionErrorThreshold( model, perfTestDataDir, runs ):

  errors = []
  for run in runs:
    print(("Testing with", run))
    datadir = perfTestDataDir + "/" + run
    datasetHeader, dataset = getPerfDataset( datadir , configs.NUMBER_OF_COUNTERS )
    dataArray = getDatasetArray(dataset)
    realData = preprocessDataArray(dataArray)
    predictedData = model.predict(realData)
    reconstructErrorList = []
    datasetLen = realData.shape[0]
    for x in range(datasetLen):
      #reconstructionError = getNormalizedDistance( realData[x], predictedData[x] )
      reconstructionError = getMSE( realData[x], predictedData[x] )
      reconstructErrorList.append(reconstructionError)

  errors.append(np.mean(reconstructErrorList) + np.std(reconstructErrorList))

  return (np.mean(errors) + np.std(errors))

"""
Run inference with dataset for test runs(executions) in the Datadir, write results in outFile
and return number of runs found as anomalous
"""
def testAutoencoder( model, perfTestDataDir, runs, outFile, threshold_error ):
  anomalousRunCount = 0
  for run in runs:
    print(("Testing with", run))
    datadir = perfTestDataDir + "/" + run

    datasetHeader, dataset = getPerfDataset( datadir , configs.NUMBER_OF_COUNTERS )
    dataArray = getDatasetArray(dataset)
    dataArray = preprocessDataArray(dataArray)
    reconstructedData, anomalyCount = runTrainedAutoencoder( model, dataArray, datasetHeader, threshold_error, outFile )
    if anomalyCount > 0 :
	    print("Anomaly found in execution ", run, " in", anomalyCount, " samples", file=outFile)
	    anomalousRunCount += 1

  return anomalousRunCount

"""
Return list of tuple(run,total_sample, anomalous_sample, ranking)
"""
def testAnomaly( model, testDataDir, runs, threshold_error ):
  test_summary = []
  for run in runs:
    datadir = testDataDir + "/" + run
    datasetHeader, dataset = getPerfDataset( datadir , configs.NUMBER_OF_COUNTERS )
    dataArray = getDatasetArray(dataset)
    dataArray = preprocessDataArray(dataArray)
    decoded_data = keras_autoencoder.predict( model, dataArray )
    datasetLen, anomalyCount, ranking = detectAnomalyPoints(dataArray, decoded_data, datasetHeader, threshold_error)
    if anomalyCount > datasetLen * configs.PERCENT_SAMPLE_FOR_ANOMALY : ##TODO: use a thresold, small % of anomaly can me ignored
      test_summary.append(AnomalyTuple(run = run, sample_count=datasetLen, anomalous_sample_count=anomalyCount, ranking = ranking))
  return test_summary

"""
Run trained autoencoder and detect anomalous samples based on 'thresoldLoss'
and write output in 'outFile'
"""
def runTrainedAutoencoder( model, testDataArray, datasetHeader, thresholdLoss, outFile ):

  decoded_data = keras_autoencoder.predict( model, testDataArray )
  dataLen, anomalyCount, ranking = detectAnomalyPoints(testDataArray, decoded_data, datasetHeader, thresholdLoss)

  return decoded_data, anomalyCount

"""
Preprocessing of data
"""
def preprocessDataArray( dataset ):
  #zero centering
  #mean_vector = np.mean(dataset, axis=0)
  #processed_dataset = dataset - mean_vector

  #normalize
  #dataset /= np.std(dataset, axis=0)

  ##normalize with max in each column : not working some 0 values?? why not avoid zeros
  datasetMax = np.max(dataset, axis=0)
  processed_dataset = np.nan_to_num(np.true_divide(dataset,datasetMax))

  return processed_dataset

"""
Util function for aggregating data files from all profile runs
"""
def getTrainDataSequence(dataDir, testDir=None, validationDir=None ):
  runs = os.listdir(dataDir) ##no sequence of directory assigned, it should not matter

  print(("Total execution/directory found for training data:" , len(runs)))
  return runs

"""
Util function for analyzing dataset
"""
def analyzeVariationInData( dataDir, testDir=None, validationDir=None ):

  runs = os.listdir(dataDir)
  results = []
  baseDataset = None
  for counter, run in enumerate(runs):
    datadir = dataDir + "/" + run
    datasetHeader, dataset = getPerfDataset( datadir , configs.NUMBER_OF_COUNTERS )
    if counter == 0:
      baseDataset = getDatasetArray(dataset)
      results.append( (run,0))
    else:
      if counter < 50:
        #results.append(compareDataset(baseDataset, getDatasetArray(dataset)))
        results.append(( run, compareDataset(baseDataset, getDatasetArray(dataset))))

  testResults = []
  if testDir != None:
    runs = os.listdir(testDir)
  for counter, run in enumerate(runs):
    datadir = dataDir + "/" + run
    datasetHeader, dataset =  getPerfDataset( datadir, configs.NUMBER_OF_COUNTERS )
    testResults.append(compareDataset( baseDataset, getDatasetArray(dataset)))

  validationResults = []
  if validationDir != None:
    runs = os.listdir(validationDir)
  for counter, run in enumerate(runs):
    datadir = dataDir + "/" + run
    datasetHeader, dataset =  getPerfDataset( datadir, configs.NUMBER_OF_COUNTERS )
    validationResults.append(compareDataset( baseDataset, getDatasetArray(dataset)))

  sorted_results = sorted(results, key=lambda tup: tup[1], reverse=True)
  return sorted_results, testResults, validationResults

"""
I/O util for writing log of results
"""
def writeTestLog(logFile, anomalySummary):
  for tuple in anomalySummary:
    print(tuple.run, tuple.sample_count, tuple.anomalous_sample_count, file=logFile)
    ## AnomalyTuple(run = run, sample_count=dataLen, anomalous_sample_count=anomalyCount, ranking = ranking)

"""
Inferencing of model and determination of anomalies
"""
def testModel( model, threshold_error, nonAnomalousDataDir, anomalousDataDir, logFile=None ):
  print("..Testing Non-anomalous")
  negative_runs = os.listdir(nonAnomalousDataDir)
  anomaly_summary = testAnomaly( model, nonAnomalousDataDir, negative_runs, threshold_error )
  if logFile != None:
    print("\n..Testing nonAnomalousData..\n", file=logFile)
    writeTestLog(logFile, anomaly_summary)
  false_positive = len(anomaly_summary)
  true_negative = len(negative_runs) - false_positive
  print("..Testing Anomalous")
  positive_runs = os.listdir(anomalousDataDir)
  anomaly_summary = testAnomaly( model, anomalousDataDir, positive_runs, threshold_error )
  if logFile != None:
    print("\n..Testing AnomalousData..\n", file=logFile)
    writeTestLog(logFile, anomaly_summary)
  true_positive = len(anomaly_summary)
  false_negative = len(positive_runs) - true_positive
  return AccuracyTuple(true_positive=true_positive, false_negative = false_negative, false_positive=false_positive, true_negative=true_negative)

"""
Function for gathering statistics on accuracy of a model for a specific test dataset and validation dataset
"""
def testModelAccuracy( model, outFilename, threshold_error, perfTestDataDir, perfValidDataDir ):

  outFile = open(outFilename, 'w')
  ## test for anomalous dataset
  runs = os.listdir(perfTestDataDir)

  anomalousRunCount = testAutoencoder( model, perfTestDataDir, runs, outFile, threshold_error )

  truePositive = anomalousRunCount ## a
  falseNegative = len(runs) - anomalousRunCount ## b
  print("Actual Positive", len(runs), file=outFile)
  print("True Positive", anomalousRunCount, file=outFile)
  print("False Negative", len(runs) - anomalousRunCount, file=outFile)

  print(("Total run ", len(runs)))
  print(("Total anomalous run found ", anomalousRunCount))

  ##validation with correct(not anomalous)  dataset
  runs = os.listdir(perfValidDataDir)

  anomalousRunCount = testAutoencoder( model, perfValidDataDir, runs, outFile, threshold_error )

  trueNegative = len(runs) - anomalousRunCount ## d
  falsePositive = anomalousRunCount  ## c
  print("Actual Negative", len(runs), file=outFile)
  print("True Negative", len(runs) - anomalousRunCount, file=outFile)
  print("False Positive", anomalousRunCount, file=outFile)

  print(("Total run ", len(runs)))
  print(("Total anomalous run found ", anomalousRunCount))

  ##calculate F score
  precision = 0
  if truePositive+falsePositive != 0:
    precision = float(truePositive)/(truePositive+falsePositive)

  recall = float(truePositive)/(truePositive+falseNegative)
  fscore = 0
  if precision + recall != 0:
    fscore = 2* (precision*recall)/(precision+recall) ##harmonic mean of precision and recall

  print("Precision" , precision, file=outFile)
  print("Recall", recall, file=outFile)
  print("Fscore", fscore, file=outFile)
  outFile.close()

  print(("Report: ", outFilename))

"""
Aggregate all data and train with all data at once
"""

def perfAnalyzerMainTrain( perfTrainDataDir, outputDir, autoencoder, threshold_final=None , saveTrainedNetwork=False):

  model = None

  training_sequence = getTrainDataSequence(perfTrainDataDir)
  train_loss_list = []
  reconstruction_error_list = []
  validation_loss_list = []
  dataset  = []
  for train_run in training_sequence:
    datadir = perfTrainDataDir + "/" + train_run
    redundantHeader, additionalDatatset = getPerfDataset( datadir , configs.NUMBER_OF_COUNTERS )
    processed_dataset_array = preprocessDataArray(getDatasetArray(additionalDatatset))
    dataset.extend(processed_dataset_array.tolist())

  if len(dataset) < configs.NUMBER_OF_COUNTERS * 2:
    print("Not enough data for training this iteration")
    sys.exit(1)
  else:
    print(("Training dataset size", len(dataset)))
  trainingDataset = getDatasetArray(dataset)
  if model == None:
    model, train_loss, validation_loss  = keras_autoencoder.trainAutoencoder( autoencoder, trainingDataset )
    train_loss_list.extend(train_loss)
    validation_loss_list.extend(validation_loss)
  else:
    model, train_loss, validation_loss  = keras_autoencoder.trainAutoencoder( model, trainingDataset )
    train_loss_list.extend(train_loss)
    validation_loss_list.extend(validation_loss)
  if saveTrainedNetwork == True :
    model.save(outputDir + "/" + configs.MODEL_SAVED_FILE_NAME + "_"+ str(i))

  return train_loss_list, validation_loss_list

"""
Train data with one or more exeuction data in one batch
"""
def perfAnalyzerMainTrainSequence( perfTrainDataDir, outputDir, autoencoder, threshold_final=None , saveTrainedNetwork=False):

  model = None

  training_sequence = getTrainDataSequence(perfTrainDataDir)
  #for epoch in configs.EXPERIMENT_EPOCHS:
  train_loss_list = []
  reconstruction_error_list = []
  validation_loss_list = []
  #for train_run in training_sequence:
  i = 0
  while i < len(training_sequence):
    train_run = training_sequence[i]
    print(("Training with", train_run))
    datadir = perfTrainDataDir + "/" + train_run

    datasetHeader, dataset = getPerfDataset( datadir , configs.NUMBER_OF_COUNTERS )

    while i < len(training_sequence):
      print ("small input, adding more data for the batch")
      if(i+1 == len(training_sequence)):
        break ## handle error
      print(("adding ", training_sequence[i+1]))
      datadir = perfTrainDataDir + "/" + training_sequence[i+1]
      i += 1
      redundantHeader, additionalDatatset = getPerfDataset( datadir , configs.NUMBER_OF_COUNTERS )
      dataset.extend(additionalDatatset)
    i += 1
    dataArray = getDatasetArray(dataset)
    trainingDataset = preprocessDataArray(dataArray)

    if len(dataset) < configs.NUMBER_OF_COUNTERS * 2:
      print("Not enough data for training this iteration")
      break
    if model == None:
      model, train_loss, validation_loss  = keras_autoencoder.trainAutoencoder( autoencoder, trainingDataset)
      train_loss_list.extend(train_loss)
      validation_loss_list.extend(validation_loss)
    else:
      model, train_loss, validation_loss  = keras_autoencoder.trainAutoencoder( model, trainingDataset )
      train_loss_list.extend(train_loss)
      validation_loss_list.extend(validation_loss)
  if saveTrainedNetwork == True :
    model.save(outputDir + "/" + configs.MODEL_SAVED_FILE_NAME + "_"+ str(i))

  return train_loss_list, validation_loss_list

"""
Util function for determining different topologies
"""
def getRangeOfNode( n ):

  list_of_numbers = []
  for i in range( int(n/4), int(3*n/4) ):
    if i> 5: #TODO: limit ???
      list_of_numbers.append(i)
  return list_of_numbers

"""
Run inference on input data and calculate reconstrution error
@return : list of reconstrution errors
"""
def getReconstructionErrors( perfTestDataDir, model ):

  runs = os.listdir(perfTestDataDir)
  reconstructErrorList = []
  for run in runs:
    #print ("Reconstruction of execution ", run)
    datadir = perfTestDataDir + "/" + run

    datasetHeader, dataset = getPerfDataset( datadir , configs.NUMBER_OF_COUNTERS )
    dataArray = getDatasetArray(dataset)
    dataArray = preprocessDataArray(dataArray)

    datasetLen = dataArray.shape[0]
    decodedData = keras_autoencoder.predict( model, dataArray )
    for x  in range(datasetLen):
      #reconstructionError = getNormalizedDistance( realData[x], predictedData[x] )
      reconstructionError = getMSE( dataArray[x], decodedData[x] )
      #for debugging
      reconstructErrorList.append(reconstructionError)
  return reconstructErrorList

"""
Aggregate all data and train with all data at once
@return trained model
"""

def aggregateAndTrain( perfTrainDataDir, autoencoder, saveTrainedNetwork=False,outputDir=None ):

  training_sequence = getTrainDataSequence(perfTrainDataDir)
  train_loss_list = []
  reconstruction_error_list = []
  validation_loss_list = []
  dataset  = []
  for train_run in training_sequence:
    datadir = perfTrainDataDir + "/" + train_run
    redundantHeader, additionalDatatset = getPerfDataset( datadir , configs.NUMBER_OF_COUNTERS )
    processed_dataset_array = preprocessDataArray(getDatasetArray(additionalDatatset))
    dataset.extend(processed_dataset_array.tolist())

  if len(dataset) < configs.NUMBER_OF_COUNTERS * 2:
    print("Not enough data for training this iteration")
    sys.exit(1)
  else:
    print(("Training dataset size", len(dataset)))
  trainingDataset = getDatasetArray(dataset)
  model, train_loss, validation_loss  = keras_autoencoder.trainAutoencoder( autoencoder, trainingDataset )
  train_loss_list.extend(train_loss)
  validation_loss_list.extend(validation_loss)

  if saveTrainedNetwork == True :
    model.save(outputDir + "/" + configs.MODEL_SAVED_FILE_NAME + "_"+ str(i))

  return model

"""
Calculates threshold of reconstruction errors = mean + 3 * std of reconstruction errors
"""
def calcThresoldError(reconstructionErrors):
  meanVal = np.mean(reconstructionErrors)
  meanVal += ( 3 * np.std(reconstructionErrors))
  return meanVal

"""
Train and test one autoencoder model with the datasets stored in the given directories
"""
def trainAndTest( autoencoder, trainDataDir, nonAnomalousTestDir, anomalousTestDataDir, logFile=None ):
  model = aggregateAndTrain( trainDataDir, autoencoder )

  print("..Training Complete")
  datasetTrainErrors = getReconstructionErrors(trainDataDir, model)
  threshold_error = calcThresoldError(datasetTrainErrors)

  print("..Thresold determined")
  print("Threshold : ", threshold_error, file=logFile)

  test_result = testModel( model, threshold_error, nonAnomalousTestDir, anomalousTestDataDir, logFile )

  model_string = keras_autoencoder.getAutoendoerShape( model )

  return (model_string, threshold_error, test_result)

"""
Create autoencoders from lists of input length and array of layers
"""
def getTopologies( inputLen, numberOfLayers ):

  candidate_autoencoders = []
  prefix = []
  for layer in range(numberOfLayers):
    number_of_prefix = 1
    if layer > 0:
      number_of_prefix = len(prefix)
    for prefix_layer in range(number_of_prefix):
      curr_layer_nodes = 1
      if layer == 0:
        curr_layer_nodes = getRangeOfNode(inputLen)
      else:
        curr_layer_nodes = getRangeOfNode(prefix[prefix_layer][-1])
      for n in curr_layer_nodes:
        new_prefix = []
        if layer > 0:
          for nodes in prefix[prefix_layer]:
            new_prefix.append(nodes)

        #print "curr network: ", inputLen, new_prefix, n
        autoencoder = keras_autoencoder.getAutoencoder(inputLen, n, new_prefix)
        candidate_autoencoders.append(autoencoder)
        new_prefix.append(n) #add new layer nodes
        prefix.append(new_prefix) #store prefix in this layer

    prefix = prefix[number_of_prefix-1:]

  return candidate_autoencoders

"""
Train and test provided list of candidate_autoencoders
or serach all possible networks in range of NUMBER_OF_HIDDEN_LAYER_TO_SEARCH
"""
def AutoPerfMain(candidate_autoencoders=None):
  trainDataDir=sys.argv[1]
  nonAnomalousTestDataDir=sys.argv[2]
  anomalousTestDataDir=sys.argv[3]
  outputDir = sys.argv[4]
  mkdir_p(outputDir)

  if candidate_autoencoders == None :
    candidate_autoencoders = getTopologies( configs.NUMBER_OF_COUNTERS, configs.NUMBER_OF_HIDDEN_LAYER_TO_SEARCH )

  out_file = open(outputDir + "/autoperf.out", 'w')
  log_file = open(outputDir + "/autoperf.log",'w')
  print("Number of autoencoder topologies:",  len(candidate_autoencoders), file=log_file)

  print("network, error, true_positive, false_negative, true_negative, false_positive", file=out_file)

  print("Train: ", trainDataDir, file=log_file)
  print("Test(nonAnomalous): ", nonAnomalousTestDataDir, file=log_file)
  print("Test(Anomalous): ", anomalousTestDataDir, file=log_file)
  out_file.close()
  log_file.close()

  for i in range(len(candidate_autoencoders)):
    out_file = open(outputDir + "/autoperf.out", 'a')
    log_file = open(outputDir + "/autoperf.log", 'a')
    print(("..Autoencder topology: ", keras_autoencoder.getAutoendoerShape(candidate_autoencoders[i])))
    print("\n..Autoencder topology: ", keras_autoencoder.getAutoendoerShape(candidate_autoencoders[i]), file=log_file)
    output = trainAndTest( candidate_autoencoders[i], trainDataDir, nonAnomalousTestDataDir, anomalousTestDataDir, log_file )

    ##output -> (model_string, threshold_error, test_result)
    print(output[0], output[1], output[2].true_positive, output[2].false_negative, output[2].true_negative, output[2].false_positive, file=out_file)
    out_file.close()
    log_file.close()

  print(("..Output to file ", outputDir+"/autoperf.out"))
  print(("..Log file ", outputDir+"/autoperf.log"))

if __name__ == "__main__" :

  if(len(sys.argv) < 4):
    print("Usage: autoperf.py path/to/trainingdata path/to/noAnomalousTestData path/to/anomalousTestData path/to/output")
    sys.exit()

  input_dim = configs.NUMBER_OF_COUNTERS
  #hidden_dims = [[ 16, 8 ], [16], [8], [16,8,4]]
  #encoding_dims = [4, 8, 4, 2]
  hidden_dims = [[ 16 ]]
  encoding_dims = [8]

  candidate_autoencoders = []
  for hidden_dim, encoding_dim in zip(hidden_dims, encoding_dims):
    autoencoder = keras_autoencoder.getAutoencoder(input_dim,  encoding_dim, hidden_dim)
    candidate_autoencoders.append(autoencoder)
  AutoPerfMain(candidate_autoencoders)
  #AutoPerfMain()


import numpy as np

def angular_distance(ra1, dec1, ra2, dec2):
    """
    Returns the angular distance between two points, two sets of points, or a set of points and one point.

    :param ra1: array or float, longitude of first point(s)
    :param dec1: array or float, latitude of first point(s)
    :param ra2: array or float, longitude of second point(s)
    :param dec2: array or float, latitude of second point(s)
    :return: angular distance(s) in degrees
    """

    # Vincenty formula, slower than the Haversine formula in some cases, but stable also at antipodes

    lon1 = np.deg2rad(ra1)
    lat1 = np.deg2rad(dec1)
    lon2 = np.deg2rad(ra2)
    lat2 = np.deg2rad(dec2)

    sdlon = np.sin(lon2 - lon1)
    cdlon = np.cos(lon2 - lon1)
    slat1 = np.sin(lat1)
    slat2 = np.sin(lat2)
    clat1 = np.cos(lat1)
    clat2 = np.cos(lat2)

    num1 = clat2 * sdlon
    num2 = clat1 * slat2 - slat1 * clat2 * cdlon
    denominator = slat1 * slat2 + clat1 * clat2 * cdlon

    return np.rad2deg(np.arctan2(np.sqrt(num1 ** 2 + num2 ** 2), denominator))


# -*- coding: utf-8 -*-
"""
Created on Mon Feb 20 20:46:21 2017

@author: ricardojdferreira
"""
import pandas as pd

"""
main module
"""
if __name__ == '__main__':

    #process energy production values
    df_produce = pd.read_excel("data/energy produce total.xlsx")
    columns_produce = list(df_produce.columns.values)

    unpiv_df_produce = pd.melt(df_produce,id_vars = columns_produce[0], var_name = 'Year', value_name = 'Value_Produce')
    unpiv_df_produce.rename(columns = {"Energy production, total (toe)":"Country"}, inplace=True)

    #process use values
    df_use = pd.read_excel("data/energy use total.xlsx")
    columns_use = list(df_use.columns.values)

    unpiv_df_use = pd.melt(df_use,id_vars = columns_use[0], var_name = 'Year', value_name = 'Value_Use')
    unpiv_df_use.rename(columns = {"Energe use, total (toe)":"Country"}, inplace=True)

    #compute data frame for visualization
    result_frame = pd.merge(unpiv_df_produce, unpiv_df_use, how = "outer", left_on = ["Country","Year"], right_on = ["Country","Year"])

    #save to .csv
    result_frame["Value_Produce"] = result_frame["Value_Produce"] * 1000
    result_frame.to_excel("data/energy_tableau.xlsx")


from selenium import webdriver
from selenium.webdriver.chrome.options import Options

def main():
    chrome_options = Options()
    chrome_options.add_argument("--disable-extensions")
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--headless")
    chrome_options.headless = True  # also works
    driver = webdriver.Chrome(executable_path='/Users/jorgecorradi/Downloads/chromedriver', options=chrome_options)
    driver.get('https://www.infomoney.com.br/cotacoes/empresas-b3/')

    symbols = []
    for el in driver.find_elements_by_css_selector('.list-companies a'):
        symbols.append(el.text)

    with open('/Users/jorgecorradi/Desktop/stocks.csv', 'w') as csv_file:
        csv_file.write("symbol;companyName;stockType;sector;category;currency" + "\n")
        for symbol in symbols:
            driver.get('https://www.fundamentus.com.br/detalhes.php?papel=' + symbol)
            print("Scraping " + symbol)

            try:
                data = {}
                data['"symbol"'] = symbol

                type = driver.find_element_by_css_selector(
                    'body > div.center > div.conteudo.clearfix > table:nth-child(2) > tbody > tr:nth-child(1) > td.label.w15 > span.txt').text

                if type == 'Papel':
                    data['companyName'] = driver.find_element_by_css_selector(
                        'body > div.center > div.conteudo.clearfix > table:nth-child(2) > tbody > tr:nth-child(3) > td:nth-child(2) > span').text

                    data['stockType'] = driver.find_element_by_css_selector(
                        'body > div.center > div.conteudo.clearfix > table:nth-child(2) > tbody > tr:nth-child(2) > td:nth-child(2) > span').text
                    data['sector'] = driver.find_element_by_css_selector(
                        'body > div.center > div.conteudo.clearfix > table:nth-child(2) > tbody > tr:nth-child(4) > td:nth-child(2) > span > a').text
                    data['category'] = 'STOCK'
                else:
                    data['companyName'] = driver.find_element_by_css_selector(
                        'body > div.center > div.conteudo.clearfix > table:nth-child(3) > tbody > tr:nth-child(2) > td:nth-child(2) > span').text

                    data['stockType'] = 'FII'
                    data['sector'] = driver.find_element_by_css_selector(
                        'body > div.center > div.conteudo.clearfix > table:nth-child(3) > tbody > tr:nth-child(4) > td:nth-child(2) > span > a').text
                    data['category'] = 'REF'

                data['currency'] = 'BRL'

                csv_file.write(";".join(data.values()) + "\n")
                csv_file.flush()
            except:
                print("couldn't get info for " + symbol)

if __name__ == '__main__':
    main()
    print("Done Scraping")


from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from django.core.mail import EmailMessage

import logging
logger = logging.getLogger(__name__)

from notifications.models import Subscription

FROM_EMAIL = getattr(settings, 'NOTIFICATION_FROM_EMAIL', 'notifications@wheresyourtrash.com')

class Command(BaseCommand):
    help = 'Run through subscriptions sending out notifications'

    def handle(self, *args, **options):
        # Gather all subscriptions that need to go out
        #subs = Subscription.objects.live()
        for sub in Subscription.objects.all():
            if sub.day_of_pickup:
                logger.info('Sending notification to {0}'.format(sub.user))
                pickup_type = sub.district.district_type.lower() # + '_notification'
                if sub.subscription_type == 'SMS':
                    recipient = sub.clean_phone_number + '@' + sub.service_provider.email_root
                else:
                    recipient = sub.user.email

                if sub.district.district_type == 'TRASH':
                    pickup_str = 'trash'
                else:
                    pickup_str = 'recycling'
                txt_content = 'Is your {0} outside and ready? The folks from {1} are going to be picking it up later today!!'.format(pickup_str, sub.district.municipality.name)
                email = EmailMessage("Where's Your Trash?", txt_content, FROM_EMAIL, [recipient])
                email.send()

                '''
                send_templated_mail(template_name=template_name,
                                    from_email=FROM_EMAIL,
                                    recipient_list=[recipient],
                                    context={'subscription':sub})
                '''


#!/usr/bin/env python

import ROOT
import os,copy
from python.TreePlotter import TreePlotter
from python.MergedPlotter import MergedPlotter
from python.myStackPlotter import StackPlotter
from python.SimplePlot import *
from python.SetCuts import SetCuts

##--- Setup: inputs
indir="./METSkim_v4"; outdir="output/newcut/"
CheckDir(outdir) # if not, this will create it
onlyStats=False;scaleDphi=False;LogY=True;doRatio=False;
sigk=1000; lumi=2.318278305;
lepsf='llnunu_l1_l1_lepsf*llnunu_l1_l2_lepsf'
triggersf='triggersf'
setcuts = SetCuts()
cuts = setcuts.abcdCuts(channel="inclusive", whichRegion="SR", isPreSelect=True, zpt_cut='100', met_cut='50')
##--- Initialize: plotter
zjetsPlotters=[]
zjetsSamples = ['DYJetsToLL_M50_BIG'] # M50_BIG = M50 + M50_Ext

for sample in zjetsSamples:
    zjetsPlotters.append(TreePlotter(indir+'/'+sample+'.root','tree'))
    if not onlyStats:
        zjetsPlotters[-1].addCorrectionFactor('1./SumWeights','tree')
        zjetsPlotters[-1].addCorrectionFactor('xsec','tree')
        #zjetsPlotters[-1].addCorrectionFactor('(1921.8*3)','xsec')
        zjetsPlotters[-1].addCorrectionFactor('genWeight','tree')
        zjetsPlotters[-1].addCorrectionFactor('puWeight','tree')
        zjetsPlotters[-1].addCorrectionFactor(triggersf,'tree')
        zjetsPlotters[-1].addCorrectionFactor(lepsf,'tree')
        if scaleDphi:
            zjetsPlotters[-1].addCorrectionFactor('dphi_sf','tree')# to scale dphi shape in BCD regions as the one in regA
ZJets = MergedPlotter(zjetsPlotters)
ZJets.setFillProperties(1001,ROOT.kGreen+2)

sigPoints=[600, 800, 1000, 1200, 1400, 1600, 1800,
           2000, 2500, 3000, 3500, 4000, 4500]
sigPlotters=[]
sigSamples = [ 'BulkGravToZZToZlepZinv_narrow_'+str(point) for point in sigPoints ]
sigSampleNames = [ str(sigk) + ' x BulkG-' + str(mass) for mass in sigPoints]

# sigXsec = {
#     'BulkGravToZZToZlepZinv_narrow_800'  : 4.42472e-04*sigk,
#     'BulkGravToZZToZlepZinv_narrow_1000' : 1.33926e-04*sigk,
#     'BulkGravToZZToZlepZinv_narrow_1200' : 4.76544e-05*sigk,
# }
for sample in sigSamples:
    sigPlotters.append(TreePlotter(indir+'/'+sample+'.root','tree'))
    if not onlyStats:
        sigPlotters[-1].addCorrectionFactor('1./SumWeights','tree')
        #sigPlotters[-1].addCorrectionFactor(str(sigXsec[sample]),'tree')
        sigPlotters[-1].addCorrectionFactor('genWeight','tree')
        sigPlotters[-1].addCorrectionFactor('puWeight','tree')
        sigPlotters[-1].addCorrectionFactor(triggersf,'tree')
        sigPlotters[-1].addCorrectionFactor(lepsf,'tree')
        sigPlotters[-1].setFillProperties(0,ROOT.kWhite)

Stack = StackPlotter()
Stack.addPlotter(ZJets, "ZJets","Z+Jets", "background")
for i in range(len(sigSamples)):
    sigPlotters[i].setLineProperties(2,ROOT.kRed+i,2)
    Stack.addPlotter(sigPlotters[i],sigSamples[i],sigSampleNames[i],'signal')

Stack.setLog(LogY)
Stack.doRatio(doRatio)

##--- Execute: plotting
ROOT.gROOT.ProcessLine('.x ../src/tdrstyle.C')
Stack.drawComp('(llnunu_l1_pt+met_para)/llnunu_l1_pt', cuts,
               10, -5.0, 5.0, titlex = "(p_{T}^{Z}+E_{T,para}^{miss})/p_{T}^{Z}", units = "GeV",
               output=outdir+"newcut.pdf")


"""
This problem was asked by Google.

Given an array of integers where every integer occurs three times except for one integer, which only occurs once,
find and return the non-duplicated integer.

For example, given [6, 1, 3, 3, 3, 6, 6], return 1. Given [13, 19, 13, 13], return 19.

Do this in O(N) time and O(1) space.

"""

def non_duplicate(arr):
    set_arr = {}

    for num in arr:
        if num in set_arr:
            set_arr[num] += 1
        else:
            set_arr[num] = 1
    print(set_arr)
    return [key for key, val in set_arr.items() if val == 1]

print(non_duplicate([6, 1, 3, 3, 3, 6, 6]))


# coding=UTF-8
#读取config.ini配置信息中的数据

import os
import codecs
import configparser
#将读取配置信息ini的过程封装成一个类
proDir = os.path.split(os.path.realpath(__file__))[0]    #该文件的地址，这里相当于这里的接口测试文档框架路径,绝对路径
configPath = os.path.join(proDir, "config.ini")         #存放在根目录下文件名是config.ini的文件，配置文件的完整地址

class ReadConfig():
    def __init__(self):
        fd = open(configPath ,encoding="utf-8")          #打开配置文件
        data = fd.read()               #读取配置文件

        #  remove BOM                  #判断文件中是否有数据
        if data[:3] == codecs.BOM_UTF8:
            data = data[3:]
            file = codecs.open(configPath, "w")
            file.write(data)
            file.close()
        fd.close()

        self.cf = configparser.ConfigParser()   #调用读取配置模块的类
        self.cf.read(configPath,encoding="utf-8")                #读取文件

#获取emall、http、database分组下指定的name值
    def get_email(self, name):
        value = self.cf.get("EMAIL", name)
        return value

    def get_http(self, name):
        value = self.cf.get("HTTP", name)
        return value

    def get_headers(self, name):
        value = self.cf.get("HEADERS", name)
        return value

    def set_headers(self, name, value):
        self.cf.set("HEADERS", name, value)
        with open(configPath, 'w+') as f:
            self.cf.write(f)

    def get_url(self, name):
        value = self.cf.get("URL", name)
        return value

    def get_db(self, name):
        value = self.cf.get("DATABASE", name)
        return value

#修改http分组下指定name的值value   现阶段还未使用
    def set_http(self,name,value):
        cfg = self.cf.set("HTTP",name,value)
        fp = open(r'config.ini','w')
        cfg.write(fp)


from base.socialRecommender import SocialRecommender
from scipy.sparse import *
import numpy as np
from numpy import linalg as LA
from joblib import Parallel, delayed
from math import sqrt
EPS = 1e-8
# this model refers to the following paper:
# #########----  Collaborative Filtering with Social Exposure: A Modular Approach to Social Recommendation   ----#############
# SEREC_boost
class SERec(SocialRecommender):
    def __init__(self,conf,trainingSet=None,testSet=None,relation=None,fold='[1]'):
        super(SERec, self).__init__(conf,trainingSet,testSet,relation,fold)

    def initModel(self):
        super(SERec, self).initModel()
        self.lam_theta = 1e-5
        self.lam_beta = 1e-5
        self.lam_y = 0.01
        self.init_mu = 0.01
        self.a = 1.0
        self.b = 99.0
        self.s= 2.2
        self.init_std = 0.5
        self.theta = self.init_std * \
            np.random.randn(self.num_users, self.emb_size).astype(np.float32)
        self.beta = self.init_std * \
            np.random.randn(self.num_items, self.emb_size).astype(np.float32)
        self.mu = self.init_mu * np.ones((self.num_users,self.num_items), dtype=np.float32)
        self.n_jobs=4
        self.batch_size=1000
        row,col,val = [],[],[]
        for user in self.data.trainSet_u:
            for item in self.data.trainSet_u[user]:
                u = self.data.user[user]
                i = self.data.item[item]
                row.append(u)
                col.append(i)
                val.append(1)

        self.X = csr_matrix((np.array(val),(np.array(row),np.array(col))),(self.num_users,self.num_items))
        row,col,val = [],[],[]
        for user in self.social.followees:
            for f in self.social.followees[user]:
                u = self.data.user[user]
                i = self.data.user[f]
                row.append(u)
                col.append(i)
                val.append(1)
        self.T = csr_matrix((np.array(val), (np.array(row), np.array(col))), (self.num_users, self.num_users))

    def trainModel(self):
        print('training...')
        self._update(self.X)

    def _update(self, X):
        '''Model training and evaluation on validation set'''
        n_users = X.shape[0]
        XT = X.T.tocsr()  # pre-compute this
        self.vad_ndcg = -np.inf
        for i in range(self.maxEpoch):

            print('epoch #%d' % i)
            self._update_factors(X, XT)
            print(self.mu)
            self._update_expo(X, n_users)

    def _update_factors(self, X, XT):
        '''Update user and item collaborative factors with ALS'''
        self.theta = recompute_factors(self.beta, self.theta, X,
                                       self.lam_theta / self.lam_y,
                                       self.lam_y,
                                       self.mu,
                                       self.n_jobs,
                                       batch_size=self.batch_size)

        self.beta = recompute_factors(self.theta, self.beta, XT,
                                      self.lam_beta / self.lam_y,
                                      self.lam_y,
                                      self.mu,
                                      self.n_jobs,
                                      batch_size=self.batch_size)
    def _update_expo(self, X, n_users):
        '''Update exposure prior'''
        print('\tUpdating exposure prior...')
        start_idx = list(range(0, n_users, self.batch_size))
        end_idx = start_idx[1:] + [n_users]
        A_sum = np.zeros(self.num_items)
        for lo, hi in zip(start_idx, end_idx):
            A_sum += a_row_batch(X[lo:hi], self.theta[lo:hi], self.beta,
                                 self.lam_y, self.mu[lo:hi]).sum(axis=0)
        A_sum=np.tile(A_sum,[self.num_users,1])
        S_sum = self.T.dot(A_sum)
        self.mu = (self.a + A_sum +(self.s-1)*S_sum- 1) / (self.a + self.b + (self.s-1)*S_sum+n_users - 2)

    def predictForRanking(self,u):
        'invoked to rank all the items for the user'
        if self.data.containsUser(u):
            u = self.data.getUserId(u)
            return self.beta.dot(self.theta[u])
        else:
            return [self.data.globalMean] * self.num_items

def get_row(Y, i):
    '''Given a scipy.sparse.csr_matrix Y, get the values and indices of the
    non-zero values in i_th row'''
    lo, hi = Y.indptr[i], Y.indptr[i + 1]
    return Y.data[lo:hi], Y.indices[lo:hi]

def a_row_batch(Y_batch, theta_batch, beta, lam_y, mu):
    '''Compute the posterior of exposure latent variables A by batch'''
    pEX = sqrt(lam_y / 2 / np.pi) * \
          np.exp(-lam_y * theta_batch.dot(beta.T) ** 2 / 2)

    A = (pEX + EPS) / (pEX + EPS + (1 - mu) / mu)
    A[Y_batch.nonzero()] = 1.
    return A

def _solve(k, A_k, X, Y, f, lam, lam_y, mu):
    '''Update one single factor'''
    s_u, i_u = get_row(Y, k)
    a = np.dot(s_u * A_k[i_u], X[i_u])
    B = X.T.dot(A_k[:, np.newaxis] * X) + lam * np.eye(f)
    return LA.solve(B, a)

def _solve_batch(lo, hi, X, X_old_batch, Y, m, f, lam, lam_y, mu):
    '''Update factors by batch, will eventually call _solve() on each factor to
    keep the parallel process busy'''
    assert X_old_batch.shape[0] == hi - lo

    if mu.shape[1] == X.shape[0]:  # update users
        A_batch = a_row_batch(Y[lo:hi], X_old_batch, X, lam_y, mu[lo:hi])
    else:  # update items
        A_batch = a_row_batch(Y[lo:hi], X_old_batch, X, lam_y, mu.T[lo:hi])

    X_batch = np.empty_like(X_old_batch, dtype=X_old_batch.dtype)
    for ib, k in enumerate(range(lo, hi)):
        X_batch[ib] = _solve(k, A_batch[ib], X, Y, f, lam, lam_y, mu)
    return X_batch

def recompute_factors(X, X_old, Y, lam, lam_y, mu, n_jobs, batch_size=1000):
    '''Regress X to Y with exposure matrix (computed on-the-fly with X_old) and
    ridge term lam by embarrassingly parallelization. All the comments below
    are in the view of computing user factors'''
    m, n = Y.shape  # m = number of users, n = number of items
    assert X.shape[0] == n
    assert X_old.shape[0] == m
    f = X.shape[1]  # f = number of factors

    start_idx = list(range(0, m, batch_size))
    end_idx = start_idx[1:] + [m]
    res = Parallel(n_jobs=n_jobs)(delayed(_solve_batch)(
        lo, hi, X, X_old[lo:hi], Y, m, f, lam, lam_y, mu)
                                   for lo, hi in zip(start_idx, end_idx))
    X_new = np.vstack(res)
    return X_new


import scrapy
import subprocess
import json
import os

def installPackage(package):
    try:
        return __import__(str(package))
    except ImportError:
        subprocess.call(["pip3", "install", "scrapy"])

# Make a dicrectory to contain JSON file and HTML file
class makeDir():
    def JSON_dir(self):
        JSON_path = os.path.join(os.getcwd(), "JSON Image file/")

        if not os.path.exists(JSON_path):
            os.mkdir(JSON_path)

        return JSON_path

    def HTML_dir(self):
        HTML_path = JSON_path = os.path.join(os.getcwd(), "HTML File/")

        if not os.path.exists(HTML_path):
            os.mkdir(HTML_path)

        return HTML_path

dir = makeDir()
JSON_directory = dir.JSON_dir()
HTML_directory = dir.HTML_dir()

# For dantri.com
class covid_dantri(scrapy.Spider):
    name = "covid_dantri"

    def start_requests(self):
        urls = [
            "https://dantri.com.vn/suc-khoe/dai-dich-covid-19.htm",
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        page = response.url.split("/")[-1].split(".")[0].split(".")[0]
        filename = f'{HTML_directory}dantri-{page}.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')

        # Writing source and caption of image into JSON file
        dantri_covid = {'img': []}

        for image in response.css("img"):
            dantri_covid['img'].append({
                "source": image.attrib["src"],
                "caption": image.attrib["alt"],
            })

        with open(f"{JSON_directory}/image_dantri.json", "w") as image_file:
            json.dump(dantri_covid, image_file)

# For tuoitre.net
class covid_tuoitre(scrapy.Spider):
    name = "covid_tuoitre"

    def start_requests(self):
        urls = [
            "https://tuoitre.vn/phong-chong-covid-19-e583.htm",
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        page = response.url.split("/")[-1].split(".")[0].split(".")[0]
        filename = f'{HTML_directory}tuoitre-{page}.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')

        tuoitre_covid = {'img': []}

        source = response.css('img::attr(src)').extract()
        caption = response.css('img::attr(alt)').extract()

        for i in range(0, len(caption)):
            tuoitre_covid['img'].append({
                "source": source[i],
                "caption": caption[i],
            })

        with open(f"{JSON_directory}image_tuoitre.json", "w") as image_file:
            json.dump(tuoitre_covid, image_file)

# For vnexpress.net
class covid_vnexpress(scrapy.Spider):
    name = "covid_vnexpress"

    def start_requests(self):
        urls = [
            "https://vnexpress.net/covid-19/tin-tuc",
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        page = response.url.split("/")[-2]
        filename = f'{HTML_directory}vnexpress-{page}.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')

        # Writing source and caption of image into JSON file
        vnexpress_covid = {'img': []}

        for image in response.css("img"):
            vnexpress_covid['img'].append({
                "source": image.attrib["src"],
                "caption": image.attrib["alt"],
            })

        with open(f"{JSON_directory}image_vnexpress.json", "w") as image_file:
            json.dump(vnexpress_covid, image_file)

def main():
    # Install Scrapy if not have
    installPackage(scrapy)

    # Crawl image and caption about Covid19 News
    # From dantri.com
    subprocess.call(["scrapy", "crawl", "covid_dantri"])

    # From tuoitre.net
    subprocess.call(["scrapy", "crawl", "covid_tuoitre"])

    # From vnexpress.net
    subprocess.call(["scrapy", "crawl", "covid_vnexpress"])

if __name__ == "__main__":
    main()


"""Ciphers module.
Provide Asymmetric and Symmetric cipher functions.\
"""
from enum import IntEnum

from Crypto import Random
from Crypto.Cipher import PKCS1_v1_5, AES
from Crypto.PublicKey import RSA
from Crypto.Random import random as rand
from Crypto.Util import Counter

class AESBlockCipher:

    class Size(IntEnum):
        bit_128 = 1
        bit_192 = 2
        bit_256 = 3

    supported_mode = [AES.MODE_CBC, AES.MODE_CTR]

    def __init__(self, keysize=Size.bit_256, mode=AES.MODE_CBC, secret=None, salt=None):
        """Generate AES key according to the keysize and secret.
        keysize -- int, AES keysize in byte (16, 24, 32).
        secret -- bytes, only used for Password-Based AES cipher.
        """

        if mode not in self.supported_mode:
            raise None  # TODO raise error

        self.aes = None
        self.MODE = mode

        if keysize not in list(map(int, self.Size)):
            raise Exception("Invalid key size.")

        keysize = int((128 + 64 * (keysize - 1)) / 8)

        if not secret:
            """Generate randome key"""
            self.key = Random.get_random_bytes(keysize)
        else:  # Password-Based
            """Construct AES key from user secret with argon2 hash function"""
            import argon2

            self.salt = Random.get_random_bytes(16) if salt is None else salt
            time_cost = 2
            memory_cost = 512
            parallelism = 2
            hash_len = keysize

            self.key = argon2.low_level.hash_secret_raw(secret, self.salt, time_cost,
                                                        memory_cost, parallelism,
                                                        hash_len, argon2.low_level.Type.I)

    def _setup_cipher(self, iv=None):
        # 16-byte, 128-bit initial vector/value for both CBC and CTR mode
        iv = Random.get_random_bytes(AES.block_size) if not iv else iv
        if self.MODE == AES.MODE_CBC:
            return iv, AES.new(self.key, AES.MODE_CBC, iv)
        if self.MODE == AES.MODE_CTR:
            ctr = Counter.new(128, initial_value=int.from_bytes(iv, byteorder="big"))
            return iv, AES.new(self.key, AES.MODE_CTR, counter=ctr)

    @staticmethod
    def pad(b):
        return b + \
               ((AES.block_size - len(b) % AES.block_size) *
                chr(AES.block_size - len(b) % AES.block_size)).encode()

    @staticmethod
    def unpad(b):
        return b[:-b[-1]]

    # Encrypt a chuck of bytes
    def encrypt(self, b, last=False):
        iv = bytes()
        if not self.aes:  # first time, setup iv
            iv, self.aes = self._setup_cipher()

        if not last:
            return iv + self.aes.encrypt(b)
        else:
            # if last chunk, pad the chunk
            ret = self.aes.encrypt(self.pad(b))
            self.aes = None
            return iv + ret  # return iv also since it could be the only chunk

    def decrypt(self, b, last=False):
        if not self.aes:  # first time, retrieve iv
            iv = b[:AES.block_size]  # both modes have the same iv size
            b = b[len(iv):]
            _, self.aes = self._setup_cipher(iv)

        if not last:
            return self.aes.decrypt(b)
        else:
            # if last chunk, unpad the chunk
            dec = self.aes.decrypt(b)
            return self.unpad(dec)

class RSACipher:
    rsa = None
    cipher = None

    class Size(IntEnum):
        bit_1024 = 1
        bit_2048 = 2
        bit_4096 = 3
        bit_8192 = 4

    def import_key(self, key, passphrase=None):
        # if key is bytes, load it directly
        if isinstance(key, bytes):
            self.rsa = RSA.importKey(key, passphrase)
        else:  # else read it as file
            with open(key, "rb") as f:
                self.rsa = RSA.importKey(f.read(), passphrase)

        self.cipher = PKCS1_v1_5.new(self.rsa)

    def export_key(self, keyfile=None, format="PEM", passphrase=None, public_only=False):
        if not hasattr(self, "rsa"):
            raise Exception("Missing RSA cipher...")

        # if has keyfile, write to key file
        if keyfile:
            with open(keyfile, "wb") as f:
                if public_only:
                    pubkey = self.rsa.publickey()
                    f.write(pubkey.exportKey(format=format))
                else:
                    f.write(self.rsa.exportKey(format=format, passphrase=passphrase))
        else:  # else return bytes
            if public_only:
                pubkey = self.rsa.publickey()
                return pubkey.exportKey(format=format)
            else:
                return self.rsa.exportKey(format=format, passphrase=passphrase)

    def generate(self, keysize=Size.bit_4096, progress_func=None):
        self.progress_func = progress_func

        if keysize not in list(map(int, self.Size)):
            raise Exception("Invalid key size.")

        keysize = {1: 1024, 2: 2048, 3: 4096, 4: 8192}[keysize]
        self.rsa = RSA.generate(keysize, progress_func=self.gen_prog)

        self.cipher = PKCS1_v1_5.new(self.rsa)

        self.gen_prog("Finished, key size: " + str(RSACipher.Size(keysize)))

    # progress function for generating key
    def gen_prog(self, s):
        d = {"p,q": "Generating p and q primes...",
             "u": "Generating public exponent...",
             "d": "Generating private exponent..."}
        if self.progress_func:
            if s in d.keys():
                self.progress_func(d[s])
            else:
                self.progress_func(s)

    # encrypt with rsa key, data size should not exceed n - 11
    def encrypt(self, b):
        if not hasattr(self, "rsa"):
            raise Exception("Missing RSA cipher...")

        return self.cipher.encrypt(b)

    def decrypt(self, b):
        if not hasattr(self, "rsa"):
            raise Exception("Missing RSA cipher...")

        return self.cipher.decrypt(b, Random.get_random_bytes(rand.randint(0, self.rsa.size())))


from yowsup.layers.interface import YowInterfaceLayer, ProtocolEntityCallback
from yowsup.layers.protocol_messages.protocolentities import TextMessageProtocolEntity
from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity
from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity

class EchoLayer(YowInterfaceLayer):
    @ProtocolEntityCallback("message")
    def onMessage(self, messageProtocolEntity):
        # send receipt otherwise we keep receiving the same message over and over

        if True:
            receipt = OutgoingReceiptProtocolEntity(messageProtocolEntity.getId(), messageProtocolEntity.getFrom(),
                                                    'read', messageProtocolEntity.getParticipant())

            outgoingMessageProtocolEntity = TextMessageProtocolEntity(
                messageProtocolEntity.getBody(),
                to=messageProtocolEntity.getFrom())

            self.toLower(receipt)
            self.toLower(outgoingMessageProtocolEntity)

    @ProtocolEntityCallback("receipt")
    def onReceipt(self, entity):
        ack = OutgoingAckProtocolEntity(entity.getId(), "receipt", entity.getType(), entity.getFrom())
        self.toLower(ack)


import cv2
import numpy as np

filename = 'cup.jpg'

print('顯示圖片')
image = cv2.imread(filename, -1)

shape = image.shape
h = shape[0]    #高
w = shape[1]    #寬
h, w, d = image.shape   #d為dimension d=3 全彩 d=1 灰階
print("寬 = ", w, ", 高 = ", h, ", D = ", d)

image = cv2.resize(image, (403, 302))
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (5, 5), 0)
circles = cv2.HoughCircles(
    gray,
    cv2.HOUGH_GRADIENT, # 偵測方法目前只支援這個參數
    1,      # 1代表偵測圖與輸入圖大小一致，填1即可
    20,     # 各圓心間的最小距離，設太小容易誤判，太大會將數個圓當成一個
    None,   # 固定填 None
    10,     # canny演算法的高閾值，此值一半為低閾值
    75,     # 超過此閾值才會被當作圓
    3,      # 最小圓半徑
    75      # 最大圓半徑
)

circles = circles.astype(int)
if len(circles) > 0:
    out = image.copy()
    for x, y, r in circles[0]:
        # 畫圓
        cv2.circle(out, (x, y), r, (0, 0, 255), 3)
        # 畫圓心
        cv2.circle(out, (x, y), 2, (0, 255, 0), 3)
    image = cv2.hconcat([image, out])

cv2.namedWindow('image', cv2.WINDOW_NORMAL)
cv2.imshow('image', image) #顯示圖片

print('在此等待任意鍵繼續, 繼續後刪除本視窗')
cv2.waitKey()
cv2.destroyAllWindows()


#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2022 Pi-Yueh Chuang <pychuang@gwu.edu>
#
# Distributed under terms of the BSD 3-Clause license.

"""Total losses versus training iterations.
"""
import sys
import pathlib
from matplotlib import pyplot

# find helpers
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1].joinpath("modulus")))
from helpers.tbreader import read_tensorboard_data  # pylint: disable=import-error  # noqa: E402

# matplotlib configuration
pyplot.rcParams.update({
    "text.usetex": True,
    "font.family": "serif",
    "font.serif": ["P052", "Pagella", "Palatino", "Palatino Linotype", "Times New Roman"],
    "figure.constrained_layout.use": True,
})

# directories
rootdir = pathlib.Path(__file__).resolve().parents[1]
modulusdir = rootdir.joinpath("modulus", "cylinder-2d-re200-zero-ic")
rootdir.joinpath("figures").mkdir(exist_ok=True)

# read data
data1 = read_tensorboard_data(modulusdir.joinpath("nn_256", "outputs"))
data2 = read_tensorboard_data(modulusdir.joinpath("nn_512", "outputs"))

# plot
fig, ax = pyplot.subplots(1, 1, figsize=(6, 3), dpi=166)
fig.suptitle("Convergence history: total loss v.s. iteration")
ax.semilogy(data1["step"], data1["loss"].ewm(span=10).mean(), label="256 neurons per layer")
ax.semilogy(data2["step"], data2["loss"].ewm(span=10).mean(), label="512 neurons per layer")
ax.set_xlabel("Iteration")
ax.set_ylabel("Total loss")
ax.legend(loc=0)
fig.savefig(rootdir.joinpath("figures", "cylinder-pinn-training-convergence.png"), bbox_inches="tight", dpi=166)


# -*- coding: utf-8 -*-

import requests
from bs4 import BeautifulSoup
from docx import Document
from docx.shared import Cm

class Exceptions:

	def signal_exception(self, reason):
		print(reason)

class Philpapers:

	def check_urls(self, raw_urls):

		urls_count = raw_urls.count('https://philpapers.org/rec/')
		urls_processing = raw_urls
		urls_list = []

		if urls_count == 0:
			exception_reason = "Brak poprawnych adresów URL odnoszących do Philpapers"
			Exceptions.signal_exception(exception_reason)

		for i in range(urls_count):
			nth_url = urls_processing[urls_processing.rfind('https://philpapers.org/rec/'):]
			urls_processing = urls_processing[0:len(urls_processing)-len(nth_url)]
			nth_url = nth_url.strip(', ')
			urls_list.append(nth_url)

		urls_list[:] = list(set(urls_list))

		return urls_list

	def get_data(self, philpapers_url):

		r_lines_dict = []

		print("ppapers open")
		r = requests.get(philpapers_url, stream=True)

		_delimiter = 'Options'

		for line in r.iter_lines():
			_line = line.decode('utf-8','ignore')
			if _line != _delimiter:
				r_lines_dict.append(_line)
			else:
				break

		r.close()
		print("closed")

		r_lines_str = ''.join(r_lines_dict)

		return r_lines_str

	def check_integrity(self, raw_data, raw_data_url):

		# if raw_data.find('schema.org/Article') > 0:
			# Philpapers.paper_is_article = True
		# elif raw_data.find('schema.org/Book') > 0:
			# Philpapers.paper_is_book = True
		if raw_data.count('schema.org/Book') == 0 and raw_data.count('schema.org/Article') == 0:
			integrity = False
		else:
			integrity = True

		return integrity

	def parse_data(self, raw_data):

		paper_is_book = False
		paper_is_article = False
		article_in_book = False
		article_in_journal = False
		paper_type = None
		paper_data = None

		# ten czek łatwiej zrobić jak jeszcze wszystko jest stringiem

		if raw_data.find('schema.org/Article') > 0:
			paper_is_article = True
		elif raw_data.find('schema.org/Book') > 0:
			paper_is_book = True

		r_bs = BeautifulSoup(raw_data, 'html.parser')

		# to się dzieje w meta:

		bs_authors_list = r_bs.find_all(attrs={'name':'citation_author'})
		authors_number = len(bs_authors_list)

		bs_title = str(r_bs.find_all(attrs={'name':'citation_title'}))
		bs_date = str(r_bs.find_all(attrs={'name':'citation_publication_date'}))

		paper_title = bs_title[bs_title.find('content=')+9:bs_title.rfind('name=')-2]
		paper_date = bs_date[bs_date.find('content=')+9:bs_date.rfind('name=')-2]

		if authors_number == 1:
			temp_author = str(bs_authors_list[0])
			temp_author = temp_author[temp_author.find('content=')+9:temp_author.rfind('name=')-2]
			temp_author_lastname = temp_author[temp_author.rfind(' '):].strip()
			temp_author_firstname = temp_author[:temp_author.rfind(' ')].strip()
			paper_authors = temp_author_lastname + ' ' + temp_author_firstname
		else:
			temp_authors_list = []
			for item in bs_authors_list:
				temp_author = str(item)
				temp_author = temp_author[temp_author.find('content=')+9:temp_author.rfind('name=')-2]
				temp_author_lastname = temp_author[temp_author.rfind(' '):].strip()
				temp_author_firstname = temp_author[:temp_author.rfind(' ')].strip()
				temp_author = temp_author_lastname + ' ' + temp_author_firstname
				temp_authors_list.append(temp_author)
			paper_authors = ', '.join(temp_authors_list)

		# to się dzieje w recAuthors:

		bs_recauthors = str(r_bs.find_all(class_='recAuthors'))

		if bs_recauthors.count('(ed.)') == 1:
			paper_authors = paper_authors + ' (ed.)'
		elif bs_recauthors.count('(eds.)') == 1:
			paper_authors = paper_authors + ' (eds.)'

		# to się dzieje w pubinfo:

		bs_pubinfo = str(r_bs.find_all(class_='recPubInfo'))

		if bs_pubinfo.find('asearch.pl') > 0:
			article_in_journal = True
		else:
			article_in_book = True

		if paper_is_book:
			paper_publisher = bs_pubinfo[bs_pubinfo.find('recPubInfo')+12:bs_pubinfo.find('copyrightYear')-17].strip()
			paper_type = "book"
			paper_data = {'authors':paper_authors, 'title':paper_title, 'publisher':paper_publisher, 'year':paper_date}
		elif paper_is_article and article_in_journal:
			temp_journal_data = bs_pubinfo[bs_pubinfo.find('asearch.pl'):bs_pubinfo.find('copyrightYear')-17]
			paper_journal_name = temp_journal_data[temp_journal_data.find('>')+1:temp_journal_data.rfind('a>')-2].strip()
			paper_journal_issue = temp_journal_data[temp_journal_data.find('em>')+3:temp_journal_data.rfind(':')].strip()
			paper_journal_pages = temp_journal_data[temp_journal_data.rfind(':')+1:].strip()
			paper_journal_pages = paper_journal_pages.replace(' ','')
			paper_journal_pages = paper_journal_pages.replace('--','-')
			paper_type = "article"
			paper_data = {'authors':paper_authors, 'title':paper_title, 'journal':paper_journal_name, 'issue':paper_journal_issue, 'pages':paper_journal_pages, 'year':paper_date}
		elif paper_is_article and article_in_book:
			temp_book_data = bs_pubinfo[bs_pubinfo.find('recPubInfo')+12:bs_pubinfo.find('copyrightYear')-17]
			book_authors = temp_book_data[temp_book_data.find('In')+2:temp_book_data.find('<em')].strip(', ')
			book_title = temp_book_data[temp_book_data.find('<em')+4:temp_book_data.rfind('em>')-2].strip()
			temp_book_publisher = temp_book_data[temp_book_data.rfind('em>')+4:temp_book_data.rfind('copyrightYear')-17]

			if temp_book_data.count(' pp. ') > 0:
				book_publisher = temp_book_data[temp_book_data.rfind('em>')+4:temp_book_data.find(' pp. ')].strip(' .')
				book_pages = temp_book_data[temp_book_data.find(' pp. ')+5:].strip(' .')
				book_pages = book_pages.replace(' ','')
				book_pages = book_pages.replace('--','-')
				paper_type = "chapter"
				paper_data = {'authors':paper_authors, 'title':paper_title, 'editors':book_authors, 'booktitle':book_title, 'publisher':book_publisher, 'pages':book_pages, 'year':paper_date}
			else:
				book_publisher = temp_book_data[temp_book_data.rfind('em>')+4:].strip('  .')
				book_pages = '(brak wyszczególnionych stron)'
				paper_type = "chapter"
				paper_data = {'authors':paper_authors, 'title':paper_title, 'editors':book_authors, 'booktitle':book_title, 'publisher':book_publisher, 'pages':book_pages, 'year':paper_date}

		return paper_type, paper_data

class MakeDocx:

	def save_document(self, doc_to_save, docx_hashed_list_name):

		doc_to_save.save(docx_hashed_list_name + '.docx')
		print('docx saved')

	def format_document(self, doc_to_format):

		# standardowe wymiary a4
		standard_height = Cm(29.7)
		standard_width = Cm(21)
		standard_margin = Cm(2.5)

		section = doc_to_format.sections[0]

		section.page_height = standard_height
		section.page_width = standard_width
		section.bottom_margin = standard_margin
		section.top_margin = standard_margin
		section.left_margin = standard_margin
		section.right_margin = standard_margin

	def iter_paragraphs(self, entry_list, hashed_list_name):

		document_to_iter = Document()

		for entry in entry_list:
			print('iter_paragraphs entry:')
			print(entry)
			document_to_iter = MakeDocx.add_paragraph_apa(self, entry[0], entry[1], document_to_iter)
			print('przerobione')

		MakeDocx.format_document(self, document_to_iter)
		MakeDocx.save_document(self, document_to_iter, hashed_list_name)
		print('przerob gotowy')

	def add_paragraph_apa(self, entry_type, entry_data, doc):

		import html

		if entry_type == 'article':

			entry_par = doc.add_paragraph()

			entry_authors = entry_par.add_run(entry_data['authors']+' ')
			entry_year = entry_par.add_run('('+entry_data['year']+'). ')
			entry_title = entry_par.add_run(entry_data['title']+'. ')
			entry_journal = entry_par.add_run(entry_data['journal']).italic = True
			entry_issue = entry_par.add_run(', '+entry_data['issue']+', ')
			entry_pages = entry_par.add_run(entry_data['pages']+'.')

			#  Nazwisko, X., Nazwisko2, X. Y., Nazwisko3, Z. (rok). Tytuł artykułu. Tytuł Czasopisma, nr rocznika(nr zeszytu), strona początku–strona końca.

		elif entry_type == 'book':

			entry_par = doc.add_paragraph()

			entry_authors = entry_par.add_run(entry_data['authors']+' ')
			entry_year = entry_par.add_run('('+entry_data['year']+'). ')
			entry_title = entry_par.add_run(entry_data['title']+'. ').italic = True
			entry_publisher = entry_par.add_run(entry_data['publisher']+'.')

			#  Nazwisko, X., Nazwisko, X. Y. (rok). Tytuł książki. Miejsce wydania: Wydawnictwo.

		elif entry_type == 'chapter':

			entry_par = doc.add_paragraph()

			entry_authors = entry_par.add_run(entry_data['authors']+' ')
			entry_year = entry_par.add_run('('+entry_data['year']+'). ')
			entry_title = entry_par.add_run(entry_data['title']+'. W: ')
			entry_editors = entry_par.add_run(entry_data['editors']+', ')
			entry_booktitle = entry_par.add_run(entry_data['booktitle']).italic = True
			entry_pages = entry_par.add_run(' (s. '+entry_data['pages']+'). ')
			entry_publisher = entry_par.add_run(entry_data['publisher']+'.')

			#  Nazwisko, X. (rok). Tytuł rozdziału. W: Y. Nazwisko, B. Nazwisko (red.), Tytuł książki (s. strona początku–strona końca). Miejsce wydania: Wydawnictwo.

		return doc

def mainloop(input_urls):
	ppapers_load = Philpapers()
	makedocx_load = MakeDocx()
	exceptions_load = Exceptions()

	urls_checked = Philpapers.check_urls(ppapers_load, input_urls)
	if len(urls_checked) > 0:

		philpapers_entries_list = []
		for item in urls_checked:
			entry_data = Philpapers.get_data(ppapers_load, item)
			integrity_check = Philpapers.check_integrity(ppapers_load, entry_data, item)
			print(integrity_check)
			if integrity_check:
				philpapers_entries_list.append(Philpapers.parse_data(ppapers_load, entry_data))
			else:
				exception_reason = 'Nie można pobrać danych z %s : odnośnik nie zawiera poprawnego schematu strony Philpapers (brak typu dokumentu). Popraw listę adresów i spróbuj jeszcze raz' % (item)
				exceptions_load.Exceptions(exceptions_load, exception_reason)

		# MakeDocx.iter_paragraphs(philpapers_entries_list)
	return philpapers_entries_list

def testloop(input_list, hashname):
	ppapers_load = Philpapers()
	makedocx_load = MakeDocx()
	exceptions_load = Exceptions()

	if len(input_list) > 0:
		philpapers_entries_list = input_list
		'''
		for item in urls_checked:
			entry_data = Philpapers.get_data(ppapers_load, item)
			integrity_check = Philpapers.check_integrity(ppapers_load, entry_data, item)
			print(integrity_check)
			if integrity_check:
				philpapers_entries_list.append(Philpapers.parse_data(ppapers_load, entry_data))
			else:
				exception_reason = 'Nie można pobrać danych z %s : odnośnik nie zawiera poprawnego schematu strony Philpapers (brak typu dokumentu). Popraw listę adresów i spróbuj jeszcze raz' % (item)
				Exceptions.signal_exception(exceptions_load, exception_reason)
		'''
		MakeDocx.iter_paragraphs(makedocx_load, philpapers_entries_list, hashname)
	return philpapers_entries_list

if __name__ == "__main__":
	test_urls = 'https://philpapers.org/rec/BAUPAT-3 https://philpapers.org/rec/SURMCA https://philpapers.org/rec/RORTGO https://philpapers.org/rec/OPPACT https://philpapers.org/rec/SMATTT https://philpapers.org/rec/RORTGO'
	test_list = [('book', {'year': '1994', 'publisher': 'Cornell University Press', 'authors': 'Lloyd Genevieve', 'title': 'Part of Nature: Self-Knowledge in Spinoza's Ethics'}), ('article', {'pages': '342-359', 'year': '2013', 'journal': 'Deleuze Studies', 'title': 'Nomadic Ethics', 'authors': 'Braidotti Rosi', 'issue': '7 (3)'}), ('chapter', {'authors': 'Smart J. J. C.', 'booktitle': 'Contemporary Debates in Metaphysics' , 'publisher': 'Blackwell', 'title': 'The tenseless theory of time', 'year': '2008', 'pages': '226-38', 'editors': 'Theodore Sider, John Hawthorne & Dean W. Zimmerman (eds.)'})]
	testloop(str(TAG(test_list)))


import wx

#----------------------------------------------------------------------

class TestPanel(wx.Panel):
    def __init__(self, parent, log):
        self.log = log
        wx.Panel.__init__(self, parent, -1)

        dpc = wx.DatePickerCtrl(self, size=(120,-1),
                                style=wx.DP_DROPDOWN | wx.DP_SHOWCENTURY)

        sizer = wx.BoxSizer(wx.VERTICAL)
        sizer.Add(dpc, 0, wx.ALL, 50)

        self.SetSizer(sizer)

#----------------------------------------------------------------------

def runTest(frame, nb, log):
    win = TestPanel(nb, log)
    return win

#----------------------------------------------------------------------

overview = """

wx.DatePickerCtrl

This control allows the user to select a date. Unlike
wx.calendar.CalendarCtrl, which is a relatively big control,
wx.DatePickerCtrl is implemented as a small window showing the
currently selected date. The control can be edited using the keyboard,
and can also display a popup window for more user-friendly date
selection, depending on the styles used and the platform.

"""

if __name__ == '__main__':
    import sys,os
    import run
    run.main(['', os.path.basename(sys.argv[0])] + sys.argv[1:])


# -*- coding: utf-8 -*-
"""
Created on Mon Aug  5 11:57:54 2019

@author: Asus
"""

from tkinter import *
from tkinter import ttk
import tkinter as tk
import tkinter.messagebox
import pandas as pd
from delete_row import*

def delete():
    global _string,droot,csv_file,cbs
    _string = []
    for name, checkbutton in cbs.items():
        if checkbutton.var.get():
            _string.append(checkbutton['text'])
    if len(_string) == 0:
        tkinter.messagebox.showinfo("Error!","No camera selected!")
    else:
        configDelete(_string,csv_file)
        tkinter.messagebox.showinfo("INFO","Camera(s) Removed!")
    droot.destroy()

def update_text():
    global cbs,_string

    _string = []
    for name, checkbutton in cbs.items():
        if checkbutton.var.get():
            _string.append(checkbutton['text'])

def deleteCam(filepath):
    global droot,csv_file,cbs
    droot = tk.Tk()

    droot.title("DELETE CAMERAS")
    width = 420
    height = 350
    screen_width = droot.winfo_screenwidth()
    screen_height = droot.winfo_screenheight()
    x = (screen_width/2) - (width/2)
    y = (screen_height/2) - (height/2)
    droot.geometry("%dx%d+%d+%d" % (width, height, x, y))

    vscrollbar = Scrollbar(droot,width=15)
    vscrollbar.grid(row=0, column=1, sticky=N+S)
    canvas = Canvas(droot,yscrollcommand=vscrollbar.set)
    canvas.grid(row=0, column=0, sticky=N+S+E+W)

    vscrollbar.config(command=canvas.yview)
    frame = Frame(canvas)
    frame.rowconfigure(1, weight=1)
    frame.columnconfigure(1, weight=1)
    slctLabel= tk.Label(frame, text="Select cameras to be removed:", fg = "black",font = 'Calibri 11 bold')
    slctLabel.grid(column=1,pady = 5,padx=10)

    csv_file = filepath
    #csv_file = r"D:\FACE_DETECT_RECOG\video_details.csv"
    df = pd.read_csv(csv_file)

    names = (df['Name'].values)

    cbs = dict()
    for i, value in enumerate(names):
        cbs[value] = tk.Checkbutton(frame, text=value, onvalue=True,
                                offvalue=False, command=update_text)
        cbs[value].var = tk.BooleanVar(frame, value=False)
        cbs[value]['variable'] = cbs[value].var

        cbs[value].grid(column=1, row=i+2, sticky='W', padx=20)

    dltButton = Button(droot, text='Delete Cameras', width=25,height =1,fg = "orange",bg = "gray", bd=2,font = 'Calibri 14 bold', command = delete,relief="raised")
    dltButton.grid(column =0,padx = 20,pady=20)
    canvas.create_window(0, 0, anchor=NW, window=frame)
    frame.update_idletasks()

    canvas.config(scrollregion=canvas.bbox("all"))

    droot.mainloop()


import os.path as op
import glob
import numpy as np
import pandas as pd
import subprocess
import spacy
from wordfreq import zipf_frequency as word_frequency
import subprocess
try:
    spacy.load('nl_core_news_sm')
except:
    process = subprocess.Popen('python -m spacy download nl_core_news_sm'.split(), stdout=subprocess.PIPE)
    output, error = process.communicate()

# Setup paths and file names
this_path = '/' + op.join(*op.realpath(__file__).split('/')[:-1])

def setup_datapath():
    fname_datapath = '/' + this_path + '/data_path.txt'

    if not op.isfile(fname_datapath):
        data_path = input("Enter data path (or create a data_path.txt):")
        if data_path[-1] != '/':
            data_path += '/'
        with open(fname_datapath, 'w') as f:
            f.write(data_path)
    with open(fname_datapath, 'r') as f:
        data_path = f.read().strip('\n')
    return data_path

def setup_logfiles():
    fname_logfiles = '/' + this_path + '/log_files.csv'
    data_path = setup_datapath()
    if not op.isfile(fname_logfiles):
        tasks = dict(visual='Vis', auditory='Aud')
        log_files = list()

        for task in ('visual', 'auditory'):

            log_path = data_path + 'sourcedata/meg_task/'
            files = glob.glob(log_path + "*-MEG-MOUS-%s*.log" % tasks[task])

            for file in np.sort(files):
                subject = file.split('-')[0].split('/')[-1]
                log_files.append(dict(
                    subject=int(subject[1:]),
                    task=task,
                    log_id=int(file.split('-')[1]),
                    log_file=op.join('sourcedata', 'meg_task',
                                     file.split('/')[-1]),
                    meg_file=op.join('sub-' + subject, 'meg',
                                     'sub-%s_task-%s_meg.ds' % (subject, task))
                    )
                )
        log_files = pd.DataFrame(log_files)
        # Remove corrupted log
        log_files = log_files.loc[(log_files.subject != 1006) &
                                  (log_files.subject != 1017)]
        log_files.to_csv(fname_logfiles)
    return pd.read_csv(fname_logfiles)

def setup_stimuli():
    fname_stimuli = '/' + this_path + '/stimuli.csv'
    data_path = setup_datapath()
    if not op.isfile(fname_stimuli):

        source = op.join(data_path, 'stimuli', 'stimuli.txt')

        with open(source, 'r') as f:
            stimuli = f.read()
        while '  ' in stimuli:
            stimuli.replace('  ', ' ')

        # clean up
        stimuli = stimuli.split('\n')[:-1]
        stim_id = [int(s.split(' ')[0]) for s in stimuli]
        sequences = [' '.join(s.split(' ')[1:]) for s in stimuli]
        stimuli = pd.DataFrame([dict(index=idx, sequence=seq)
                                for idx, seq in zip(stim_id, sequences)])
        stimuli.to_csv(fname_stimuli, index=False)
    return pd.read_csv(fname_stimuli, index_col='index')

def setup_morphemes():
    # download
    commands = ('polyglot download morph2.en morph2.ar',
                'polyglot download morph2.nl')
    for command in commands:
        subprocess.Popen(command.split(), stdout=subprocess.PIPE)

data_path = setup_datapath()
log_files = setup_logfiles()
stimuli = setup_stimuli()

def _parse_log(log_fname):
    with open(log_fname, 'r') as f:
        text = f.read()

    # Fixes broken inputs
    text = text.replace('.\n', '.')

    # file is made of two blocks
    block1, block2 = text.split('\n\n\n')

    # read first header
    header1 = block1.replace(' ', '_').split('\n')[3].split('\t')
    header1[6] = 'time_uncertainty'
    header1[8] = 'duration_uncertainty'

    # read first data
    df1 = pd.DataFrame([s.split('\t') for s in block1.split('\n')][5:],
                       columns=header1)
    # the two dataframe are only synced on certains rows
    common_samples = ('Picture', 'Sound', 'Nothing')
    sel = df1['Event_Type'].apply(lambda x: x in common_samples)
    index = df1.loc[sel].index

    # read second header
    header2 = block2.replace(' ', '_').split('\n')[0].split('\t')
    header2[7] = 'time_uncertainty'
    header2[9] = 'duration_uncertainty'

    # read second data
    df2 = pd.DataFrame([s.split('\t') for s in block2.split('\n')[2:-1]],
                       columns=header2, index=index)

    # remove duplicate
    duplicates = np.intersect1d(df1.keys(), df2.keys())
    for key in duplicates:
        assert (df1.loc[index, key] == df2[key].fillna('')).all()
        df2.pop(key)

    log = pd.concat((df1, df2), axis=1)
    return log

def _clean_log(log):
    # Relabel condition: only applies to sample where condition changes
    translate = dict(
        ZINNEN='sentence',
        WOORDEN='word_list',
        FIX='fix',
        QUESTION='question',
        Response='response',
        ISI='isi',
        blank='blank',
    )
    for key, value in translate.items():
        sel = log.Code.astype(str).str.contains(key)
        log.loc[sel, 'condition'] = value
    log.loc[log.Code == '', 'condition'] = 'blank'

    # Annotate sequence idx and extend context to all trials
    start = 0
    block = 0
    context = 'init'
    log['new_context'] = False
    query = 'condition in ("word_list", "sentence")'
    for idx, row in log.query(query).iterrows():
        log.loc[start:idx, 'context'] = context
        log.loc[start:idx, 'block'] = block
        log.loc[idx, 'new_context'] = True
        context = row.condition
        block += 1
        start = idx
    log.loc[start:, 'context'] = context
    log.loc[start:, 'block'] = block

    # Format time
    log['time'] = 0
    idx = log.Time.str.isnumeric() == True  # noqa
    log.loc[idx, 'time'] = log.loc[idx, 'Time'].astype(float) / 1e4

    # Extract individual word
    log.loc[log.condition.isna(), 'condition'] = 'word'
    idx = log.condition == 'word'
    words = log.Code.str.strip('0123456789 ')
    log.loc[idx, 'word'] = words.loc[idx]
    sel = log.query('word=="" and condition=="word"').index
    log.loc[sel, 'word'] = pd.np.nan
    log.loc[log.word.isna() & (log.condition == "word"), 'condition'] = 'blank'
    return log

def _add_stim_id(log, verbose):
    # Find beginning of each sequence (word list or sentence)
    start = 0
    sequence_pos = -1
    for idx, row in log.query('condition == "fix"').iterrows():
        if sequence_pos >= 0:
            log.loc[start:idx, 'sequence_pos'] = sequence_pos
        sequence_pos += 1
        start = idx
    log.loc[start:, 'sequence_pos'] = sequence_pos

    # Find corresponding stimulus id
    stim_id = 0
    lower30 = lambda s: s[:30].lower()  # noqa
    stimuli['first_30_chars'] = stimuli.sequence.apply(lower30)
    sel = slice(0, 0)
    for pos, row in log.groupby('sequence_pos'):
        if pos == -1:
            continue

        # select words in this sequence
        sel = row.condition == "word"
        if not sum(sel):
            continue

        # match with stimuli
        first_30_chars = ' '.join(row.loc[sel, 'word'])[:30].lower()  # noqa
        stim_id = stimuli.query('first_30_chars == @first_30_chars').index
        assert len(stim_id) == 1
        stim_id = stim_id[0]

        n_words = len(stimuli.loc[stim_id, 'sequence'].split(' '))
        if (n_words != sum(sel)) and verbose:
            print('mistach of %i words in %s (stim %i)' % (n_words - sum(sel),
                                                           pos, stim_id))
            print('stim: %s' % stimuli.loc[stim_id, 'sequence'])
            print('log: %s' % ' '.join(row.loc[sel, 'word']))

        # Update
        log.loc[row.index, 'stim_id'] = stim_id
    return log

def read_log(log_fname, task='auto', verbose=False):
    log = _parse_log(log_fname)
    log = _clean_log(log)
    if task == 'auto':
        task = 'visual' if log_fname[-7:] == 'Vis.log' else 'auditory'
    if task == 'visual':
        # TODO: add sequence annotation for auditory
        log = _add_stim_id(log, verbose=verbose)
    return log

def get_log_times(log, events, sfreq):
    sel = np.sort(np.r_[
        np.where(events[:, 2] == 20)[0],  # fixation
        np.where(events[:, 2] == 10)[0]  # context
    ])
    common_megs = events[sel]
    common_logs = log.query('(new_context == True) or condition=="fix"')

    last_log = common_logs.time[0]
    last_meg = common_megs[0, 0]
    last_idx = 0
    assert len(common_megs) == len(common_logs)
    for common_meg, (idx, common_log) in zip(
            common_megs, common_logs.iterrows()):

        if common_meg[2] == 20:
            assert common_log.condition == 'fix'
        else:
            assert common_log.condition in ('sentence', 'word_list')

        log.loc[idx, 'meg_time'] = common_meg[0] / sfreq

        sel = slice(last_idx+1, idx)
        times = log.loc[sel, 'time'] - last_log + last_meg / sfreq
        assert np.all(np.isfinite(times))
        log.loc[sel, 'meg_time'] = times

        last_log = common_log.time
        last_meg = common_meg[0]
        last_idx = idx

        assert np.isfinite(last_log) * np.isfinite(last_meg)

    # last block
    sel = slice(last_idx+1, None)
    times = log.loc[sel, 'time'] - last_log + last_meg / sfreq
    log.loc[sel, 'meg_time'] = times
    log['meg_sample'] = np.array(log.meg_time.values * sfreq, int)
    return log

def match_list(A, B, on_replace='delete'):
    """Match two lists of different sizes and return corresponding indice
    Parameters
    ----------
    A: list | array, shape (n,)
        The values of the first list
    B: list | array: shape (m, )
        The values of the second list
    Returns
    -------
    A_idx : array
        The indices of the A list that match those of the B
    B_idx : array
        The indices of the B list that match those of the A
    """
    from Levenshtein import editops

    unique = np.unique(np.r_[A, B])
    label_encoder = dict((k, v) for v, k in enumerate(unique))

    def int_to_unicode(array):
        return ''.join([str(chr(label_encoder[ii])) for ii in array])

    changes = editops(int_to_unicode(A), int_to_unicode(B))
    B_sel = np.arange(len(B)).astype(float)
    A_sel = np.arange(len(A)).astype(float)
    for type, val_a, val_b in changes:
        if type == 'insert':
            B_sel[val_b] = np.nan
        elif type == 'delete':
            A_sel[val_a] = np.nan
        elif on_replace == 'delete':
            # print('delete replace')
            A_sel[val_a] = np.nan
            B_sel[val_b] = np.nan
        elif on_replace == 'keep':
            # print('keep replace')
            pass
        else:
            raise NotImplementedError
    B_sel = B_sel[np.where(~np.isnan(B_sel))]
    A_sel = A_sel[np.where(~np.isnan(A_sel))]
    assert len(B_sel) == len(A_sel)
    return A_sel.astype(int), B_sel.astype(int)

def add_part_of_speech(df):
    sentences = ' '.join(df['word'].values)
    while '  ' in sentences:
        sentences = sentences.replace('  ', ' ')

    nlp = spacy.load("nl_core_news_sm")
    doc = [i for i in nlp(sentences)]
    from_idx, to_idx = match_list([i.text for i in doc],
                                  df.word.values)

    part_of_speech = [doc[i].pos_ for i in from_idx]
    idx = df.index.values[to_idx]
    df.loc[idx, 'pos'] = part_of_speech
    df.loc[df.pos == 'X', 'pos'] = pd.np.nan
    #categories = ['ADJ', 'ADP', 'ADV', 'CONJ', 'DET',
                  # 'NOUN', 'NUM', 'PRON', 'PROPN', 'VERB']
    #df = df.join(pd.get_dummies(df.pos, columns=categories))
    df = df.join(pd.get_dummies(df.pos))
    if 'PROPN' not in df.keys():
        verb_idx = df.columns.get_loc("VERB")
        df.insert(verb_idx, 'PROPN', 0)

    return df

def add_word_frequency(df):
    freq = df.word.apply(lambda word: word_frequency(word, 'nl'))  # noqa
    df['word_freq'] = freq
    return df

def add_word_length(df):
    df['word_length'] = df.word.astype(str).apply(len)
    return df

def add_letter_count(df):
    word = df.word.str
    for letter in 'abcdefghijklmnopqrstuvwxyz':
        df[letter] = word.count(letter)
    return df

def read_mri_events(event_fname):
    """This is needs to be enriched depending on the analysis
    """
    # Read MRI events
    events = pd.read_csv(event_fname, sep='\t')

    # Add context: sentence or word list?
    contexts = dict(WOORDEN='word_list', ZINNEN='sentence')
    for key, value in contexts.items():
        sel = events.value.str.contains(key)
        events.loc[sel, 'context'] = value
        events.loc[sel, 'condition'] = value

    # Clean up MRI event mess
    sel = ~events.context.isna()
    start = 0
    context = 'init'
    for idx, row in events.loc[sel].iterrows():
        events.loc[start:idx, 'context'] = context
        start = idx
        context = row.context
    events.loc[start:, 'context'] = context

    # Add event condition: word, blank, inter stimulus interval etc
    conditions = (('50', 'pulse'), ('blank', 'blank'), ('ISI', 'isi'))
    for key, value in conditions:
        sel = events.value == key
        events.loc[sel, 'condition'] = value

    events.loc[events.value.str.contains('FIX '), 'condition'] = 'fix'

    # Extract words from file
    sel = events.condition.isna()
    words = events.loc[sel, 'value'].apply(lambda s: s.strip('0123456789 '))
    events.loc[sel, 'word'] = words

    # Remove empty words
    sel = (events.word.astype(str).apply(len) == 0) & (events.condition.isna())
    events.loc[sel, 'word'] = pd.np.nan
    events.loc[sel, 'condition'] = 'blank'
    events.loc[~events.word.isna(), 'condition'] = 'word'

    # --- Add word frequency
    sel = events.condition == 'word'

    def get_word_freq(word):
        return word_frequency(word, 'en', wordlist='best', minimum=0.0)

    events.loc[sel, 'word_freq'] = events.loc[sel, 'word'].apply(get_word_freq)

    # --- Add word length
    sel = events.condition == 'word'
    events['word_length'] = events.word.astype(str).apply(len)

    # --- TODO Add whatever features may be relevant here

    return events


# coding: utf-8
import enum
from core.network.timeout import Timeout

class EntityTypes(enum.Enum):
    LOADCASE = "loadcase"
    SIMULATION = "simulation"
    TASK = "task"
    STYPE = "submodelType"
    SUBMODEL = "submodel"

class AbstractEntity(object):
    """
    Class containing common behaviour of main CML-Bench entities, such as loadcases, simulations, tasks
    """
    def __init__(self, app_session, identifier):
        self._app_session = app_session
        self._identifier = identifier
        self._http_session = self._app_session.session
        self._sender = self._app_session.sender
        self._handler = self._app_session.handler

        self._entity_type = None
        self._name = None
        self._parent_id = None
        self._tree_path = None
        self._tree_id = None

    def __repr__(self):
        return "Entity type: {} | Entity ID: {}".format(self.entity_type, self.identifier)

    @property
    def entity_type(self):
        return self._entity_type

    @property
    def identifier(self):
        return self._identifier

    @property
    def name(self):
        return self._name

    @property
    def parent_id(self):
        return self._parent_id

    @property
    def tree_path(self):
        return self._tree_path

    @property
    def tree_id(self):
        return self._tree_id

    def _set_entity_type(self, entity_type):
        if isinstance(entity_type, EntityTypes):
            self._entity_type = entity_type

    def _setup_attributes(self):
        if self.entity_type:
            response = self._sender.send_entity_base_info_request(self.identifier, self.entity_type.value)
            Timeout.hold_your_horses()
            self._handler.set_response(response)
            base_info = self._handler.handle_response_to_entity_base_info_request()
            self._name = base_info.get("name")
            self._parent_id = base_info.get("parent_id")
            self._tree_path = base_info.get("tree_path")
            self._tree_id = base_info.get("tree_id")

class Loadcase(AbstractEntity):
    """
    Class for representation of the loadcase entity
    """
    def __init__(self, app_session, identifier):
        super().__init__(app_session, identifier)
        self._set_entity_type(EntityTypes.LOADCASE)
        self._setup_attributes()

    def get_list_of_simulations(self):
        """
        Method for getting a list of simulations, belonging to the loadcase
        :return: list of simulation objects, or None if some error occurred during reading simulations
        """
        response = self._sender.send_loadcase_simulations_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        simulation_ids_list = self._handler.handle_response_to_loadcase_simulations_request()
        if simulation_ids_list:
            simulations = []
            for simulation_id in simulation_ids_list:
                simulations.append(Simulation(self._app_session, simulation_id))
            return simulations
        return None

class Simulation(AbstractEntity):
    """
    Class for representation of the simulation entity
    """
    def __init__(self, app_session, identifier):
        super().__init__(app_session, identifier)
        self._set_entity_type(EntityTypes.SIMULATION)
        self._setup_attributes()

    def clone(self):
        """
        Method for creating a new simulation, based on the current one
        :return: id of the new simulation, or None if failed to clone simulation
        """
        response = self._sender.send_clone_simulation_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        return self._handler.handle_response_to_clone_simulation_request()

    def get_parent_loadcase(self):
        """
        Method for getting a parent loadcase
        :return: loadcase object
        """
        return Loadcase(self._app_session, self.parent_id)

    def get_list_of_tasks(self):
        """
        Method for getting a list of tasks, belonging to the simulation
        :return: list of task objects, of None if some error occurred during reading tasks
        """
        response = self._sender.send_simulation_tasks_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        simulation_tasks_list = self._handler.handle_response_to_simulation_tasks_request()
        if simulation_tasks_list:
            tasks = []
            for task_id in simulation_tasks_list:
                tasks.append(Task(self._app_session, task_id))
            return tasks
        return None

    def get_list_of_submodels(self):
        response = self._sender.send_simulation_submodels_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        simulation_submodels_list = self._handler.handle_response_to_simulation_submodels_request()
        if simulation_submodels_list:
            submodels = []
            for submodel_id in simulation_submodels_list:
                submodels.append(Submodel(self._app_session, submodel_id))
            return submodels
        return None

    def add_new_sumbodels(self, new_submodel_ids):
        # First, get list of current submodels
        response = self._sender.send_simulation_submodels_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        simulation_submodels_list = self._handler.handle_response_to_simulation_submodels_request()
        if not simulation_submodels_list:
            simulation_submodels_list = []
        # Append new submodel to the list of existing submodels
        simulation_submodels_list.extend(new_submodel_ids)
        # Send request to update simulation submodels (that's how it works in CML-Bench)
        response = self._sender.send_simulation_submodels_update_request(self.identifier, simulation_submodels_list)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        simulation_submodels_list = self._handler.handle_response_to_simulation_submodels_request()
        if simulation_submodels_list:
            submodels = []
            for submodel_id in simulation_submodels_list:
                submodels.append(Submodel(self._app_session, submodel_id))
            return submodels
        return None

class Task(AbstractEntity):
    """
    Class for representation of the task entity
    """
    def __init__(self, app_session, identifier):
        super().__init__(app_session, identifier)
        self._set_entity_type(EntityTypes.TASK)
        self._setup_attributes()

    def get_status(self):
        response = self._sender.send_task_status_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        return self._handler.handle_response_to_task_status_response()

class SubmodelType(AbstractEntity):
    """
    Class for representation of the s|type entity
    """
    def __init__(self, app_session, identifier):
        super().__init__(app_session, identifier)
        self._set_entity_type(EntityTypes.STYPE)
        self._setup_attributes()

    def get_list_of_submodels(self):
        response = self._sender.send_stype_submodels_request(self.tree_path)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        stype_submodels_list = self._handler.handle_response_to_stype_submodels_requests()
        if stype_submodels_list:
            submodels = []
            for submodel_id in stype_submodels_list:
                submodels.append(Submodel(self._app_session, submodel_id))
            return submodels
        return None

    def upload_new_submodel(self, *files, **params):
        if "stype" in params.keys():
            stype = SubmodelType(self._app_session, params.get("stype"))
        else:
            stype = SubmodelType(self._app_session, self._app_session.cfg.server_storage)

        if "add_to_clipboard" in params.keys():
            add_to_clipboard = "on" if bool(params.get("add_to_clipboard")) else "off"
        else:
            add_to_clipboard = "off"

        submodels = []
        for file in files:
            response = self._sender.send_upload_submodel_request(file, stype.tree_id, add_to_clipboard)
            Timeout.hold_your_horses()
            self._handler.set_response(response)
            sumbodel_id = self._handler.handle_response_to_upload_submodel_request()
            if sumbodel_id:
                submodels.append(Submodel(self._app_session, sumbodel_id))

        return submodels

class Submodel(AbstractEntity):
    """
    Class for representation of the submodel entity
    """
    def __init__(self, app_session, identifier):
        super().__init__(app_session, identifier)
        self._set_entity_type(EntityTypes.SUBMODEL)
        self._setup_attributes()