# -*- coding: utf-8 -*-
import glob

def write_file(result_f, id_f, out_f):
    for result_line, id_line in zip(result_f, id_f):
        id = id_line.rstrip().split('\t')[-1]
        result_line = id + '\t' + result_line
        out_f.write(result_line)

def get_files():
    base_path = '/home/sadayuki/Dropbox/Data-Mining-Research/'
    result_files = glob.glob(base_path + 'data_evaluation/human_results/classification_results/*')
    id_files = glob.glob(base_path + 'data_testing/manual_classification/8_shuffled/*')
    out_file  = base_path + 'data_combined_results/original/human_sentiment'
    return result_files, id_files, out_file

def get_id_file(result_file, id_files):
    result = ''
    for id_file in id_files:
        if id_file.split('/')[-1] == result_file.split('/')[-1]:
            return id_file

def combine_in_files():
    result_files, id_files, out_file = get_files()
    with open(out_file, 'w+') as out_f:
        for result_file in result_files:
            id_file = get_id_file(result_file, id_files)
            with open(result_file, 'r') as result_f, open(id_file, 'r') as id_f:
                write_file(result_f, id_f, out_f)

if __name__ == '__main__':
    combine_in_files()


import requests
from bs4 import BeautifulSoup
import pandas as pd
import json

import email_sender

COL_title = 'Title'
COL_rating = 'Rating'
COL_review_count = 'Reviews'
COL_taste_profile = 'Taste Profile'
COL_taste_profile_order = 'Taste Profile Order'
COL_image_url_and_shop_url = 'Wine'
COL_image_url_and_shop_url_separator = '||'
COL_price = '$ Normal'
COL_discounted_price = '$ Discount'
COL_final_discount_percent = '% Discount'
COL_offer_type = 'Offer'

def get_number_pages(rode_wijn_page):
    li_wijn_rood = rode_wijn_page.find("li", {"class": "wijn-rood"})
    quantity_span = li_wijn_rood.find('span').find('span')
    quantity_rode_wijn = quantity_span.string[1:-1]
    pages_qty = int(int(quantity_rode_wijn) / 12)
    return pages_qty

def get_wijn_df_columns():
    return [COL_title, COL_rating, COL_review_count, COL_taste_profile,
            COL_image_url_and_shop_url, COL_price, COL_discounted_price,
            COL_final_discount_percent, COL_offer_type]

def new_wijn_df_row(title, rating, review_count, taste_profile, image_url, price, discounted_price,
                    offer_type):

    final_discounted_percent = round(((1 - float(discounted_price) / float(price)) * 100), 2)

    return [title, rating, review_count, taste_profile, image_url, price, discounted_price,
            final_discounted_percent,offer_type]

def parse_wiijn_row_from_article_tag(article):
    img = article.find('img',{'itemprop':'image'})

    title = img['alt']
    rating = article.find('ul',{'class':'rating-box'})['data-rating']
    try:
        h4 = article.find('h4')
        if h4:
            review_count = article.find('h4').contents[0]
        else:
            review_count = 0
    except:
        review_count = -2

    try:
        all_span_title = article.findAll('span',{'class':'title'})
        if len(all_span_title) > 1:
            taste_profile = all_span_title[1].string
        elif len(all_span_title) == 1:
            taste_profile = all_span_title[0].string
        else:
            taste_profile = 'not found'
    except:
        taste_profile = 'error'

    image_url = img['src'] + COL_image_url_and_shop_url_separator + 'https://www.gall.nl' + \
                           article.find('a')['href']
    price = json.loads(article['data-tracking-impression'])['price']

    sticker_title = article.find('div', {'class': 'sticker-title'})
    sticker_text = article.find('div',{'class':'sticker-text'})

    offer_type = ""
    if sticker_title:
        offer_type += str(sticker_title)

    if sticker_text:
        offer_type += str(sticker_text)

    if title.startswith('La Palma Ca') > 0:
        str(1)

    price_item_active = article.find('span', {'class': 'price item active'})
    span_price_discount = article.find('span', {'class': 'price'})

    if price_item_active:
        discounted_price = round(float(price_item_active.text),2)
    elif '2 halen, 1 betalen' in offer_type or '50% korting' in offer_type:
        discounted_price = round(float(price) / 2,2)
    elif '2e halve prijs' in offer_type:
        discounted_price = round(((float(price) / 2) + float(price)) / 2, 2)
    elif span_price_discount:
        discounted_price = round(float(span_price_discount.text), 2)
    else:
        discounted_price = price

    article.find('span',{'class':'price item active'})

    return new_wijn_df_row(
        title,
        rating,
        review_count,
        taste_profile,
        image_url,
        price,
        discounted_price,
        offer_type
    )

def get_wiijn_rows_from_page(rode_wijn_page):
    products_grid = rode_wijn_page.find('div', {'class': 'row products-grid'})
    articles = products_grid.find_all('article')
    wijn_list = []
    for article in articles:
        wijn_list.append(parse_wiijn_row_from_article_tag(article))
    return wijn_list

def get_html_style(table_html):
    return """

            <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
            <title>html title</title>

            """ + table_html+''

def get_rode_wijn_df():
    URL_template = 'https://www.gall.nl/shop/acties/?filter=categorie%3Dwijn%2Frode-wijn&page={}'

    wijn_list = []

    url = URL_template.format(1)

    response = requests.get(url)
    rode_wijn_page = BeautifulSoup(response.text, 'html.parser')

    pages_qty = get_number_pages(rode_wijn_page)
    print(str(pages_qty) + ' pages!')
    wiijn_rows = get_wiijn_rows_from_page(rode_wijn_page)
    wijn_list.extend(wiijn_rows)

    for i in range(2, pages_qty + 1):
        url = URL_template.format(i)
        response = requests.get(url)
        rode_wijn_page = BeautifulSoup(response.text, 'html.parser')
        wijn_list.extend(get_wiijn_rows_from_page(rode_wijn_page))
        print('Retrieved page ' + str(i))

    wijn_df = pd.DataFrame(wijn_list, columns=get_wijn_df_columns())
    return wijn_df

def get_rode_wijn_html(wijn_df):
    taste_profile_order = {'Royal Red': 1,
                           'Big Red': 2,
                           'Classic Red': 3,
                           'Easy Rider': 4,
                           'Fresh Red': 5,
                           'Fruit Smoothie': 6}

    wijn_df[COL_taste_profile_order] = wijn_df.apply(
        lambda row: taste_profile_order.setdefault(row[COL_taste_profile], 100), axis=1)
    wijn_df.sort_values(by=[COL_taste_profile_order, COL_rating,COL_review_count,COL_discounted_price], ascending=[True, False,False,True], inplace=True)
    wijn_df_modif_index = wijn_df.set_index([COL_taste_profile, COL_title])

    def format_img(urls):
        urls_array = urls.split(COL_image_url_and_shop_url_separator)
        img_url = urls_array[0]
        shop_url = urls_array[1]
        return ''.format(shop_url, img_url)

    def format_rating(x):
        try:
            return '{0:.2f}'.format(float(x))
        except:
            return '??? ' + str(x)

    columns_to_html = [COL_rating, COL_review_count, COL_discounted_price, COL_price, COL_final_discount_percent,
                       COL_image_url_and_shop_url,COL_offer_type]

    pd.set_option('display.max_colwidth', -1)
    table_html = wijn_df_modif_index.to_html(columns=columns_to_html,
                                       escape=False,
                                       formatters={
                                            COL_image_url_and_shop_url: format_img,
                                            COL_rating: format_rating,
                                            COL_final_discount_percent: lambda x: '{0:.0f}%'.format(x),
                                            COL_discounted_price: lambda x: '${}'.format(x),
                                            COL_price: lambda x: '${}'.format(x),
                                       })

    return get_html_style(table_html)


# SJSU CMPE 138Spring2021TEAM8
import simplejson as json
from flask import Flask, make_response, jsonify, Blueprint, request
from flask_mysql_connector import MySQL
from flask import current_app
import random
mysql = MySQL()

animals_api = Blueprint('animals_api', __name__)

@animals_api.route('/animals', methods=['GET', 'POST', 'DELETE', 'PUT'])
def animals():
    if request.method == 'GET':
        try:
            conn = mysql.connection
            cursor = conn.cursor(dictionary=True)
            if 'species_id' in request.args:
                get_animal = """SELECT * FROM Species  \
                          WHERE species_id = %s"""
                cursor.execute(get_animal, (request.args.get('species_id'),))
                current_app.logger.info("Fetching species %s",request.args.get('species_id'))
                rows = cursor.fetchone()
            else:
                get_animal = "SELECT * FROM Species"
                cursor.execute(get_animal)
                current_app.logger.info("Fetching all species in the park")
                rows = cursor.fetchall()
            return make_response(json.dumps(rows), 200)
        except Exception as e:
            current_app.logger.error(e)
            return make_response("false", 500)
        finally:
            cursor.close()
            conn.close()
    if request.method == 'POST':
        try:
            body = request.json
            post_animal = """INSERT INTO Species (species_id,name,age,description,gender,category)
            VALUES ( %s, %s, %s, %s, %s, %s )"""
            species_id =random.randint(100, 999999)
            data = (
                species_id, body['name'], body['age'], body['description'], body['gender'], body['category'])
            conn = mysql.connection
            cursor = conn.cursor(dictionary=True)
            cursor.execute(post_animal, data)
            current_app.logger.info("Inserting species %s", body['name'])
            conn.commit()
            return make_response("true", 200)
        except Exception as e:
            current_app.logger.error(e)
            return make_response("false", 500)
        finally:
            cursor.close()
            conn.close()
    if request.method == 'DELETE':
        try:
            delete_species = """DELETE FROM Species where species_id = %s"""
            data = (request.args.get('species_id'),)
            conn = mysql.connection
            cursor = conn.cursor(dictionary=True)
            cursor.execute(delete_species, data)
            current_app.logger.info("Delete species %s", request.args.get('species_id'))
            conn.commit()
            return make_response("true", 200)
        except Exception as e:
            current_app.logger.error(e)
            return make_response("false", 500)
        finally:
            cursor.close()
            conn.close()
    if request.method == 'PUT':
        try:
            body = request.json
            update_employee = """UPDATE Species set species_id = %s,name = %s,age= %s, description= %s,gender= %s,
            category = %s WHERE species_id = %s"""
            data = (
                body['species_id'], body['name'], body['age'], body['description'], body['gender'], body['category'],
                body['species_id'])
            conn = mysql.connection
            cursor = conn.cursor(dictionary=True)
            cursor.execute(update_employee, data)
            current_app.logger.info("Update species %s", body['name'])
            conn.commit()
            return make_response("true", 200)
        except Exception as e:
            current_app.logger.error(e)
            return make_response("false", 500)
        finally:
            cursor.close()
            conn.close()

@animals_api.route('/animals-stats', methods=['GET'])
def getStats():
    try:
        conn = mysql.connection
        cursor = conn.cursor(dictionary=True, buffered=True)
        stats = {}
        count_query = "SELECT COUNT(*) as total_count FROM Species;"
        cursor.execute(count_query)
        row = cursor.fetchone()
        stats['total_count'] = row['total_count']

        gender_query = """SELECT gender, COUNT(*) as count
                        FROM Species
                        GROUP BY gender;"""
        cursor.execute(gender_query)
        rows = cursor.fetchall()
        stats['gender_stats'] = rows

        category_query = """SELECT category, COUNT(*) as count
                        FROM Species
                        GROUP BY category;"""
        cursor.execute(category_query)
        rows = cursor.fetchall()
        stats['category_stats'] = rows

        age_query = """SELECT SUM(CASE WHEN age < 10 THEN 1 ELSE 0 END) AS 'Under 10',
                        SUM(CASE WHEN age BETWEEN 11 AND 20 THEN 1 ELSE 0 END) AS '11-20',
                        SUM(CASE WHEN age BETWEEN 21 AND 30 THEN 1 ELSE 0 END) AS '21-30',
                        SUM(CASE WHEN age > 30 THEN 1 ELSE 0 END) AS '30 And More'
                        FROM Species"""
        cursor.execute(age_query)
        rows = cursor.fetchall()
        stats['age_stats'] = rows
        return make_response(json.dumps(stats), 200)
    except Exception as e:
        print(e)
        return make_response("false", 500)


#%%
import os
import numpy as np
import theano
import theano.tensor as T
import matplotlib.pyplot as plt
#%%
# Data and normalization
inputs = np.array([[3,5],[5,1],[10,2]]).astype("float32")
outputs = np.array([[75],[82],[93]]).astype("float32")
inputs = inputs / np.amax(inputs, axis = 0)
outputs = outputs / 100
#%%
# hyper parameters and weights
inputLayerSize = 2
hiddenLayerSize = 3
outputLayerSize = 1
W1 = theano.shared((np.random.randn(inputLayerSize,hiddenLayerSize)).astype("float32"), name = "W1")
b1 = theano.shared( np.zeros(hiddenLayerSize).astype("float32") , name = "b1")
W2 = theano.shared((np.random.randn(hiddenLayerSize,outputLayerSize)).astype("float32"), name = "W2")
b2 = theano.shared( np.zeros(outputLayerSize).astype("float32") , name = "b2")
#%%
# Forward propogation
X = T.matrix("X")
z1 = T.dot(X,W1) + b1
a1 = T.nnet.sigmoid(z1)
z2 = T.dot(a1,W2) + b2
# using ReLu improve the results a lot
y_hat = T.nnet.relu(z2)
forward = theano.function([X], y_hat)
#%%
# cost function, gradient and optimizer
epsilon = 0.01
y = T.fcol("y")
loss = 0.5 * ((y - y_hat)**2).sum()
calloss = theano.function([X,y], loss)
# gradinet
dW1, dW2 = T.grad(loss, [W1,W2])
db1, db2 = T.grad(loss,[b1,b2])
# optimizer
#%%
train  = theano.function(
        inputs = [X,y],
        outputs = [y_hat,loss],
        updates = [
                [W2, W2 - epsilon * dW2],
                [W1, W1 - epsilon * dW1],
                [b2, b2 - epsilon * db2],
                [b1, b1 - epsilon * db1]
                ]
        )
#%%
cost = []
for i in range(20000):
    pred, cost_iter = train(inputs, outputs)
    cost.append(cost_iter)
plt.plot(cost)
print (pred,"\n" ,outputs)


from numba import cuda
import numpy as np
import math
import time

'''
Distance Matrix Calculation Accelerated by GPGPU
'''

@cuda.jit
def dist_matrix_kernel(vectors, sizes, dist_matrix):
    ## Input 1: the list of vectors
    ## Input 3: the size of each vector
    ## Output : distance matrix
    i = cuda.threadIdx.x + cuda.blockIdx.x * cuda.blockDim.x
    j = cuda.threadIdx.y + cuda.blockIdx.y * cuda.blockDim.y

    vsize = sizes[0]
    isize = sizes[1]

    if i >= isize or j >= isize:
        return
    if i == j:
        dist_matrix[i, j] = 0
        return
    else:
        dist = 0
        for idx in range(vsize):
            dist += (vectors[i, idx] - vectors[j, idx]) ** 2
        dist_matrix[i, j] = math.sqrt(dist)

## GPU Accelerated Distance matrix calculation
def dist_matrix_gpu(vectors):
    vector_size = len(vectors[0])
    item_num = len(vectors)

    ## INPUT
    vectors_global_mem = cuda.to_device(vectors)
    sizes_global_mem = cuda.to_device(np.array([vector_size, item_num]))

    ## OUTPUT
    dist_matrix_global_mem = cuda.device_array((item_num, item_num))

    ## Run Kernel
    TPB = 16
    tpb = (TPB, TPB)
    bpg = (math.ceil(item_num / TPB), math.ceil(item_num / TPB))

    dist_matrix_kernel[bpg, tpb](vectors_global_mem, sizes_global_mem, dist_matrix_global_mem)

    ## GET Result
    dist_matrix = dist_matrix_global_mem.copy_to_host()

    return dist_matrix

## CPU Distance matrix calculation (for comparison & test)
def dist_matrix_cpu(vectors):
    vector_size = len(vectors[0])
    item_num = len(vectors)

    dist_matrix = np.zeros((item_num, item_num))

    for i in range(item_num):
        for j in range(i):
            dist_matrix[i, j] = np.linalg.norm(vectors[i] - vectors[j])
            dist_matrix[j, i] = dist_matrix[i, j]

    return dist_matrix

'''
k-nearest neighbor implementation with
the further acceleration acheived by preprocessed distance matrix
'''


import numpy as np

__all__ = (
    'lower',
    'upper',
    'title',
    'capitalize',
    'swapcase',
    'length',
    'isalnum',
    'isalpha',
    'isdigit',
    'isspace',
    'islower',
    'isupper',
    'istitle',
    'isnumeric',
    'isdecimal',
    'strip',
    'lstrip',
    'rstrip',
    'center',
    'ljust',
    'rjust',
    'split',
    'rsplit',
    'partition',
    'rpartition',
    'find',
    'rfind',
    'index',
    'rindex',
    'startswith',
    'endswith',
    'concat',
    'contains',
    'repeat',
    'replace_pattern',
    # 'slice',
    # 'slice_replace',
    # 'count'
)

###################################################################################################
#                              METHODS WITH NO EXTRA PARAMETERS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param new_column: the destination column (if not set, `column` will be used)
# :return: the transformed dataframe
###################################################################################################

def _generate_basic_str_postprocess(method_name, docstring):
    def f(df, column, new_column=None):
        method = getattr(df[column].str, method_name)
        new_column = new_column or column
        df.loc[:, new_column] = method()
        return df

    f.__name__ = method_name
    f.__doc__ = f'{docstring}\n' \
                f':param df: the dataframe\n' \
                f':param column: the column\n' \
                f':param new_column: the destination column (if not set, `column` will be used)\n' \
                f':return: the transformed dataframe'
    return f

doc = 'Compute length of each string of `column`'
length = _generate_basic_str_postprocess('len', doc)

# lower, upper, capitalize, title, swapcase
###################################################################################################
doc = 'Converts all characters of `column` to lowercase.'
lower = _generate_basic_str_postprocess('lower', doc)

doc = 'Converts all characters of `column` to uppercase.'
upper = _generate_basic_str_postprocess('upper', doc)

doc = 'Converts first character to uppercase and remaining ' \
      'to lowercase for each line of `column`.'
capitalize = _generate_basic_str_postprocess('capitalize', doc)

doc = 'Converts first character to uppercase and remaining ' \
      'to lowercase for each word of each line of `column`.'
title = _generate_basic_str_postprocess('title', doc)

doc = 'Converts uppercase to lowercase and lowercase to uppercase for each word of `column`.'
swapcase = _generate_basic_str_postprocess('swapcase', doc)

# isalnum, isalpha, isdigit, isspace, islower, isupper, istitle, isnumeric, isdecimal
###################################################################################################
doc = 'Check whether all characters in each string in `column` are alphanumeric'
isalnum = _generate_basic_str_postprocess('isalnum', doc)

doc = 'Check whether all characters in each string in `column` are alphabetic'
isalpha = _generate_basic_str_postprocess('isalpha', doc)

doc = 'Check whether all characters in each string in `column` are digits'
isdigit = _generate_basic_str_postprocess('isdigit', doc)

doc = 'Check whether all characters in each string in `column` are whitespace'
isspace = _generate_basic_str_postprocess('isspace', doc)

doc = 'Check whether all characters in each string in `column` are lowercase'
islower = _generate_basic_str_postprocess('islower', doc)

doc = 'Check whether all characters in each string in `column` are uppercase'
isupper = _generate_basic_str_postprocess('isupper', doc)

doc = 'Check whether all characters in each string in `column` are titlecase'
istitle = _generate_basic_str_postprocess('istitle', doc)

doc = 'Check whether all characters in each string in `column` are numeric'
isnumeric = _generate_basic_str_postprocess('isnumeric', doc)

doc = 'Check whether all characters in each string in `column` are decimal'
isdecimal = _generate_basic_str_postprocess('isdecimal', doc)

###################################################################################################
#                                        STRIP METHODS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param to_strip: (str: None) set of characters to be removed
# :param new_column: the destination column (if not set, `column` will be used)
# :return: the transformed dataframe
###################################################################################################
def _generate_strip_str_postprocess(method_name, docstring):
    def f(df, column, *, to_strip=None, new_column=None):
        method = getattr(df[column].str, method_name)
        new_column = new_column or column
        df.loc[:, new_column] = method(to_strip)
        return df

    f.__name__ = method_name
    f.__doc__ = f'{docstring}\n' \
                f':param df: the dataframe\n' \
                f':param column: the column\n' \
                f':param to_strip: (str: None) set of characters to be removed\n' \
                f':param new_column: the destination column (if not set, `column` will be used)\n' \
                f':return: the transformed dataframe'
    return f

doc = 'Strip whitespace (including newlines) from each string in `column` from both sides'
strip = _generate_strip_str_postprocess('strip', doc)

doc = 'Strip whitespace (including newlines) from each string in `column` from left side'
lstrip = _generate_strip_str_postprocess('lstrip', doc)

doc = 'Strip whitespace (including newlines) from each string in `column` from left side'
rstrip = _generate_strip_str_postprocess('rstrip', doc)

###################################################################################################
#                              METHODS with `width` and `fillchar`
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param width: (int) minimum width
# :param fillchar: (default: \' \') additional character for filling
# :param new_column: the destination column (if not set, `column` will be used)
# :return: the transformed dataframe
###################################################################################################

def _generate_width_str_postprocess(method_name, docstring):
    def f(df, column, *, width, fillchar=' ', new_column=None):
        method = getattr(df[column].str, method_name)
        new_column = new_column or column
        df.loc[:, new_column] = method(width, fillchar=fillchar)
        return df

    f.__name__ = method_name
    f.__doc__ = f'{docstring}\n' \
                f':param df: the dataframe\n' \
                f':param column: the column\n' \
                f':param width (int): minimum width\n' \
                f':param fillchar: (default: \' \') additional character for filling\n' \
                f':param new_column: the destination column (if not set, `column` will be used)\n' \
                f':return: the transformed dataframe'
    return f

doc = 'Filling left and right side of strings in `column` with an additional character'
center = _generate_width_str_postprocess('center', doc)

doc = 'Filling right side of strings in `column` with an additional character'
ljust = _generate_width_str_postprocess('ljust', doc)

doc = 'Filling left side of strings in `column` with an additional character'
rjust = _generate_width_str_postprocess('rjust', doc)

###################################################################################################
#                                        SPLIT METHODS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param new_column: the destination column (if not set, `column` will be used)
# :param sep: (default: \' \') string or regular expression to split on
# :param limit: (default: None) limit number of splits in output
# :return: the transformed dataframe
###################################################################################################
def _generate_split_str_postprocess(method_name, docstring):
    def f(df, column, *, new_columns, sep=' ', limit=None):
        method = getattr(df[column].str, method_name)
        df_split = method(pat=sep, n=limit, expand=True)
        nb_cols = df_split.shape[1]
        df[new_columns[:nb_cols]] = df_split
        return df

    f.__name__ = method_name
    f.__doc__ = f'{docstring}\n' \
                f':param df: the dataframe\n' \
                f':param column: the column\n' \
                f':param new_column: the destination column (if not set, `column` will be used)\n' \
                f':param sep: (default: \' \') string or regular expression to split on\n' \
                f':param limit: (default: None) limit number of splits in output\n' \
                f':return: the transformed dataframe'
    return f

doc = 'Split each string in the caller’s values by given pattern, propagating NaN values'
split = _generate_split_str_postprocess('split', doc)

doc = 'Split each string `column` by the given delimiter string, ' \
      'starting at the end of the string and working to the front'
rsplit = _generate_split_str_postprocess('rsplit', doc)

###################################################################################################
#                                        PARTITION METHODS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param new_column: the destination column (if not set, `column` will be used)
# :param sep: (default: \' \') string or regular expression to split on
# :return: the transformed dataframe
###################################################################################################
def _generate_partition_str_postprocess(method_name, docstring):
    def f(df, column, *, new_columns, sep=' '):
        if len(new_columns) != 3:
            raise ValueError('`new_columns` must have 3 columns exactly')
        method = getattr(df[column].str, method_name)
        df[new_columns] = method(sep)
        return df

    f.__name__ = method_name
    f.__doc__ = f'{docstring}\n' \
                f':param df: the dataframe\n' \
                f':param column: the column\n' \
                f':param new_column: the destination column (if not set, `column` will be used)\n' \
                f':param sep: (default: \' \') string or regular expression to split on\n' \
                f':return: the transformed dataframe'
    return f

doc = 'Split the string at the first occurrence of sep, and return 3 elements containing ' \
      'the part before the separator, the separator itself, and the part after the separator. ' \
      'If the separator is not found, return 3 elements containing the string itself, ' \
      'followed by two empty strings.'
partition = _generate_partition_str_postprocess('partition', doc)

doc = 'Split the string at the last occurrence of sep, and return 3 elements containing ' \
      'the part before the separator, the separator itself, and the part after the separator. ' \
      'If the separator is not found, return 3 elements containing two empty strings, ' \
      'followed by the string itself.'
rpartition = _generate_partition_str_postprocess('rpartition', doc)

###################################################################################################
#                                   INDEX AND FIND METHODS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param new_column: the destination column (if not set, `column` will be used)
# :param sub: substring being searched
# :param start: (default: 0) left edge index
# :param end: (default: None) right edge index
# :return: the transformed dataframe
###################################################################################################
def _generate_find_str_postprocess(method_name, docstring):
    def f(df, column, *, sub, start=0, end=None, new_column=None):
        method = getattr(df[column].str, method_name)
        new_column = new_column or column
        df.loc[:, new_column] = method(sub, start, end)
        return df

    f.__name__ = method_name
    f.__doc__ = f'{docstring}\n' \
                f':param df: the dataframe\n' \
                f':param column: the column\n' \
                f':param new_column: the destination column (if not set, `column` will be used)\n' \
                f':param sub: substring being searched\n' \
                f':param start: (default: 0) left edge index\n' \
                f':param end: (default: None) right edge index\n' \
                f':return: the transformed dataframe'
    return f

doc = 'Return lowest indexes in each strings in `column` where the substring ' \
      'is fully contained between [start:end]. Return -1 on failure'
find = _generate_find_str_postprocess('find', doc)

doc = 'Return highest indexes in each strings in `column` where the substring ' \
      'is fully contained between [start:end]. Return -1 on failure'
rfind = _generate_find_str_postprocess('rfind', doc)

doc = 'Return lowest indexes in each strings where the substring is fully contained ' \
      'between [start:end]. This is the same as str.find except instead of returning -1, ' \
      'it raises a ValueError when the substring is not found'
index = _generate_find_str_postprocess('index', doc)

doc = 'Return highest indexes in each strings where the substring is fully contained ' \
      'between [start:end]. This is the same as str.find except instead of returning -1, ' \
      'it raises a ValueError when the substring is not found'
rindex = _generate_find_str_postprocess('rindex', doc)

###################################################################################################
#                                  STARTSWITH/ENDSWITH METHODS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param new_column: the destination column (if not set, `column` will be used)
# :param pat: character sequence
# :param na: (default: NaN) object shown if element tested is not a string
# :return: the transformed dataframe
###################################################################################################
def _generate_with_str_postprocess(method_name, docstring):
    def f(df, column, *, pat, na=np.nan, new_column=None):
        method = getattr(df[column].str, method_name)
        new_column = new_column or column
        df.loc[:, new_column] = method(pat, na=na)
        return df

    f.__name__ = method_name
    f.__doc__ = f'{docstring}\n' \
                f':param df: the dataframe\n' \
                f':param column: the column\n' \
                f':param new_column: the destination column (if not set, `column` will be used)\n' \
                f':param pat: character sequence\n' \
                f':param na: (default: NaN) object shown if element tested is not a string\n' \
                f':return: the transformed dataframe'
    return f

doc = 'Test if the start of each string element matches a pattern.'
startswith = _generate_with_str_postprocess('startswith', doc)

doc = 'Test if the end of each string element matches a pattern.'
endswith = _generate_with_str_postprocess('endswith', doc)

###################################################################################################
#                                        OTHER METHODS
###################################################################################################
def concat(df, *, columns, new_column, sep=None):
    """
    Concatenate `columns` element-wise
    :param df: the dataframe
    :param columns: list of columns to concatenate
    :param new_column: the destination column (if not set, `column` will be used)
    :param sep: (default: None) the separator
    :return: the transformed dataframe
    """
    if len(columns) < 2:
        raise ValueError('The `columns` parameter needs to have at least 2 columns')
    first_col, *other_cols = columns
    df.loc[:, new_column] = df[first_col].astype(str).str.cat(df[other_cols].astype(str), sep=sep)
    return df

def contains(df, column, *, pat, new_column=None, case=True, na=None, regex=True):
    """
    Test if pattern or regex is contained within strings of `column`
    :param df: the dataframe
    :param column: the column
    :param pat: (str) character sequence or regular expression.
    :param new_column: the destination column (if not set, `column` will be used)
    :param case: (bool) if True, case sensitive.
    :param na: fill value for missing values.
    :param regex: (bool) default True
    :return: the transformed dataframe
    """
    new_column = new_column or column
    df.loc[:, new_column] = df[column].str.contains(pat, case=case, na=na, regex=regex)
    return df

def repeat(df, column, *, times, new_column=None):
    """
    Duplicate each string in `column` by indicated number of time
    :param df: the dataframe
    :param column: the column
    :param times: (int) times to repeat the string
    :param new_column: the destination column (if not set, `column` will be used)
    :return: the transformed dataframe
    """
    new_column = new_column or column
    df.loc[:, new_column] = df[column].str.repeat(times)
    return df

def replace_pattern(df, column, *, pat, repl, new_column=None, case=True, regex=True):
    """
    Replace occurrences of pattern/regex in ``column` with some other string
    :param df: the dataframe
    :param column: the column
    :param pat: (str) character sequence or regular expression.
    :param repl: (str) replacement string
    :param new_column: the destination column (if not set, `column` will be used)
    :param case: (bool) if True, case sensitive.
    :param regex: (bool) default True
    :return: the transformed dataframe
    """
    new_column = new_column or column
    df.loc[:, new_column] = df[column].str.replace(pat, repl, case=case, regex=regex)
    return df


#!/usr/bin/python3
import datetime
from datetime import datetime, timedelta
import argparse
import sys
from lib.datacalc import datacalculator
from lib.checktotal import videocounter
from lib.youtubevideosearch import *
from lib.downloadcomments import *

#### VARIABLES #####

pAfter=None
pBefore=None
ptoken=""
stalkerlimit=10000

##### ARGS DATA FORMAT AND EXISTENCE VALIDATION ######

def validate_date(date):
    try:
        return datetime.strptime(date, "%Y-%m-%d").date()
    except ValueError:
        msg="Not a valid date: '{0}'".format(date)
        raise argparse.ArgumentTypeError(msg)
def validate_vorder(order):
    orders=["date","rating","relevance","title","videoCount","viewCount"]
    if order in orders:
        return order
    else:
        msg=ValueError("Invalid video order option.")
        raise argparse.ArgumentTypeError(msg)
def validate_corder(order):
    orders=["time","relevance"]
    if order in orders:
        return order
    else:
        msg=ValueError("Invalid comment order option.")
        raise argparse.ArgumentTypeError(msg)

if __name__ == '__main__':

    ########### ARGPARSE #############
    parser = argparse.ArgumentParser(prog='Youtube Stalker v.0.1', usage='%(prog)s [options]',description='Youtube Stalker is a simple python tool to help you monitor youtube videos comments about a topic of your interest and generate a output that can be used for data analytics and intelligence.',epilog='Usage example: youstalker.py -a 12-01-2018 -b 12-31-2018 -q "cats"')
    parser.add_argument('-q', '--query',help='Keyword - video search argument',action='store', required=True)
    parser.add_argument('-a', '--after', help='Published after - video search argument. Format YYYY-MM-DD', action="store", type=validate_date)
    parser.add_argument('-b', '--before',help='Published before - video search argument. Format YYYY-MM-DD', action="store", type=validate_date)
    parser.add_argument('-l','--video-limit', help='Video results limit. (default: 500). Maximum = {0}'.format(stalkerlimit), type=int, default='500')
    parser.add_argument('-o','--video-order',help='Possible orders: date, rating, relevance, title, videoCount, viewCount. (default: viewCount)',default='viewCount', action="store",type=validate_vorder)
    parser.add_argument('-L','--comment-limit', help='Comments limit for each video.(default: all comments)',type=int)
    parser.add_argument('-O','--comment-order', help='Possible orders: time, relevance. (default: time)', default='time',type=validate_corder)
    args = parser.parse_args()

    ########### ARGS VALUES VALIDATION ##########
    if args.video_limit > stalkerlimit:
        print("Stalker's limit is {0} videos for video search. Quitting...".format(stalkerlimit))
        sys.exit(1)
    if args.after and args.before:
        diff= args.after - args.before
        if diff.days > 0:
            print("--before and --after range is invalid!")
            sys.exit(1)
    if (args.video_limit > 500 and (args.after==None or args.before==None)):
        print("Youtube v3 API has a embeded video search limit of 500. \nIf you provide publishedAfter and publishedBefore values \nYoustalker will calculate the time range and split your \nquery into small queries to bypass this restriction and bring you 10000 videos results max.")
        sys.exit(1)
    if args.after:
        pAfter=(str(args.after)+"T00:00:01Z")
    if args.before:
        pBefore=(str(args.before)+"T23:59:59Z")

    ######### STALKER USE CASES ############

    if ((pAfter and pBefore) != None and args.limit > 500):
        print("Wait... I'm working on your request...")
        videototal=videocounter(args.query,pAfter,pBefore,ptoken,args.video_order)
        days=datacalculator(pAfter,pBefore,videototal)
        pega_comments(videos,args.comment_order,args.comment_limit)
        videos=youtube_searchdate(args.query,pAfter,pBefore,ptoken,args.video_order,days,args.video_limit,videototal)
    else:
        print("Wait... I'm working on your request...")
        videos=youtube_search(args.query,pAfter,pBefore,ptoken,args.video_order,args.video_limit)
        pega_comments(videos,args.comment_order,args.comment_limit)


import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import os, sys
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.dirname(CURRENT_DIR))

from torch.autograd import Variable
from torch.utils.data import TensorDataset,DataLoader,SubsetRandomSampler
from embeddings.model import SNEmbedding,PoincareDistance
from embeddings.rsgd import poincare_grad,RiemannianSGD,euclidean_retraction

class_path = 'class_dict.txt'

split_folder = ('zeroshot/standard_split/AWA2/')
split_version = 1

def get_features():

    train_split_f = open(split_folder+"trainclasses"+str(split_version)+".txt",'r')
    val_split_f = open(split_folder+"valclasses"+str(split_version)+".txt",'r')
    features_f = open("zeroshot/Animals_with_Attributes2/Features/ResNet101/AwA2-features.txt",'r')
    labels_f = open("zeroshot/Animals_with_Attributes2/Features/ResNet101/AwA2-labels.txt",'r')
    class_file = open("zeroshot/Animals_with_Attributes2/classes.txt","r")
    wordnet_file = open("wordnet_dict.txt","r")

    emb = SNEmbedding(1181,32,PoincareDistance,1.0)
    emb.load_state_dict(torch.load('mammals_32d.pth')['model'])
    for i in emb.parameters():
        weights = i

    all_classes = set([])
    train_classes = set([])
    val_classes = set([])
    ci_map = {}

    for line in wordnet_file:
        line = line[:-1].split(",")
        all_classes.add(line[1])
    wordnet_file = open("wordnet_dict.txt","r")

    c = 0
    for line in train_split_f:
        line = line[:-1]
        if line in all_classes:
            train_classes.add(line)
            ci_map[line] = c
            c += 1
    for line in val_split_f:
        line = line[:-1]
        if line in all_classes:
            val_classes.add(line)
            ci_map[line] = c
            c += 1

    class_index_map = {}
    index_class_map = {}
    emb_map = {}
    n = 0
    i_map = {}
    for line in class_file:
        line = line.strip().split("\t")
        if line[1] in train_classes or line[1] in val_classes:
            class_index_map[line[1]] = ci_map[line[1]]
            index_class_map[ci_map[line[1]]] = line[1]
            i_map[n] = ci_map[line[1]]
        n += 1

    for line in wordnet_file:
        line = line[:-1].split(",")
        if int(line[0])-1 in i_map:
            emb_map[class_index_map[line[1]]] = weights[int(line[3])].data

    train_features = []
    train_labels = []
    val_features = []
    val_labels = []

    for feature,label in zip(features_f,labels_f):
        label = int(label)-1
        if label in i_map:
            if i_map[label] in index_class_map.keys():
                if index_class_map[i_map[label]] in train_classes:
                    train_features.append(feature[:-1].split(" "))
                    train_labels.append(i_map[label])
                elif index_class_map[i_map[label]] in val_classes:
                    val_features.append(feature[:-1].split(" "))
                    val_labels.append(i_map[label])

    batch_size = 32
    train_features = np.array(train_features,dtype=np.float32)
    train_labels = np.array(train_labels,dtype=np.float32)
    val_features = np.array(val_features,dtype=np.float32)
    val_labels = np.array(val_labels,dtype=np.float32)
    trainset = TensorDataset(torch.FloatTensor(train_features),torch.LongTensor(train_labels))
    trainloader = DataLoader(trainset,batch_size=batch_size,shuffle=True)
    valset = TensorDataset(torch.FloatTensor(val_features),torch.LongTensor(val_labels))
    valloader = DataLoader(valset,batch_size=batch_size,shuffle=True)

    return trainloader,valloader,index_class_map,emb_map

class ImageEmbedding(nn.Module):
    def __init__(self,input_dim,output_dim,num_classes=39,layers=[],activation=F.relu):
        super(ImageEmbedding, self).__init__()
        self.input_dim = input_dim
        self.output_dim = output_dim
        self.activation = activation
        self.num_classes = num_classes
        self.layers = []
        d = input_dim
        for h_dim in layers:
            self.layers.append(nn.Linear(d,h_dim))
            d = h_dim
        self.layers = nn.ModuleList(self.layers)
        self.output = nn.Linear(d,output_dim)
        self.distfn = PoincareDistance()
        self.softmax = nn.LogSoftmax()

    def forward(self,x,emb_weights):
        for l in self.layers:
            x = l(x)
            x = self.activation(x)
        x = self.output(x)
        x = x/(torch.norm(x,dim=1).reshape([-1,1])+1e-6)
        stacked = torch.stack([x]*emb_weights.shape[1],1)
        dists = self.distfn(stacked,emb_weights)
        return self.softmax(-dists)

def train(model,emb_map,trainloader,valloader,lr=1e-3,n_epochs=5000,n_train=26,n_val=13,batch_size=32):

    train_emb = [torch.FloatTensor(emb_map[i]) for i in range(n_train)]
    train_emb = torch.stack(train_emb)
    val_emb = [torch.FloatTensor(emb_map[i]) for i in range(n_train,n_train+n_val)]
    val_emb = torch.stack(val_emb)

    optimizer = optim.Adam(model.parameters(),lr=lr)
    model_loss = nn.CrossEntropyLoss()

    for ep in range(n_epochs):
        print("Epoch # : " + str(ep))
        num_train = 0.0
        train_loss = 0.0
        train_acc = 0.0

        train_batch_emb = torch.stack([train_emb]*batch_size)
        train_batch_emb = Variable(train_batch_emb,requires_grad=False).cuda()
        for x,y in trainloader:
            x = x.cuda()
            y = y.cuda()
            optimizer.zero_grad()
            if x.shape[0] == batch_size:
                preds = model(x,train_batch_emb)
                loss = model_loss(preds,y)
                _, preds = torch.max(preds.data,1)
                train_acc += torch.sum(preds==y.data)
                train_loss += loss.item()
                num_train += x.shape[0]
                loss.backward(retain_graph=True)
        #             optimizer.step(lr=lr)
                optimizer.step()

        if ep % 1 == 0:
            print("Train loss : " + str(train_loss/num_train))
            print("Train accuracy : " + str(float(train_acc.item())/num_train))

            val_batch_emb = torch.stack([val_emb]*batch_size)
            val_batch_emb = Variable(val_batch_emb,requires_grad=False).cuda()
            num_val = 0.0
            val_loss = 0.0
            val_acc = 0.0
            for x,y in valloader:
                x = x.cuda()
                y = y.cuda()
                y = y-n_train
                if x.shape[0] != batch_size:
                    temp_emb = Variable(torch.stack([val_emb]*x.shape[0]),requires_grad=False).cuda()
                    preds = model(x,temp_emb)
                else:
                    preds = model(x,val_batch_emb)
                loss = model_loss(preds,y)
                val_loss += loss.item()
                num_val += x.shape[0]
                _, preds = torch.max(preds.data,1)
                val_acc += torch.sum(preds==y.data)
            print("Validation loss : " + str(val_loss/num_val))
            print("Validation accuracy : " + str(float(val_acc.item())/num_val))

trainloader,valloader,index_hash,emb_map = get_features()
model = ImageEmbedding(2048,32,layers=[256],num_classes=26).to('cuda')
train(model,emb_map,trainloader,valloader,num_classes=26)


# Assign analytes to the right workflow for library QC, based on which prep was
# used.
import sys
import re
from collections import defaultdict
from genologics.lims import *
from genologics import config

DEFAULT_WORKFLOW = "NSC QC"
WORKFLOW_MAPPING = {
        'NSC_16S': 'NSC 16S'
        }

def main(process_id):
    lims = Lims(config.BASEURI, config.USERNAME, config.PASSWORD)
    process = Process(lims, id=process_id)
    analytes = process.all_inputs(unique=True, resolve=True)

    workflow_analytes = defaultdict(list)
    # Identify workflow for each of the samples
    for analyte in analytes:
        sample_prep_used = analyte.samples[0].udf.get('Sample prep NSC')
        workflow = WORKFLOW_MAPPING.get(sample_prep_used, DEFAULT_WORKFLOW)
        workflow_analytes[workflow].append(analyte)

    # Load all workflows in the system
    workflow_data = lims.get_workflows(add_info=True)

    # Look up each workflow name in the list of workflows, and assign it if available
    for workflow_prefix, analytes in workflow_analytes.items():
        for workflow, info in zip(*workflow_data):
            if info['status'] == "ACTIVE" and info['name'].startswith(workflow_prefix):
                lims.route_analytes(analytes, workflow)
                break
        else:
            print (("Error: Unknown workflow '{}' for samples ".format(workflow_prefix) +
                    ", ".join(analyte.name for analyte in analytes) +
                    "."))
            sys.exit(1)

if __name__ == "__main__":
    main(*sys.argv[1:])


# Copyright (C) 2018 Bloomberg LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#  <http://www.apache.org/licenses/LICENSE-2.0>
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
OperationsService
=================

"""

import logging

import grpc

from google.protobuf.empty_pb2 import Empty

from buildgrid._exceptions import InvalidArgumentError
from buildgrid._protos.google.longrunning import operations_pb2_grpc, operations_pb2
from buildgrid.server._authentication import AuthContext, authorize

class OperationsService(operations_pb2_grpc.OperationsServicer):

    def __init__(self, server):
        self.__logger = logging.getLogger(__name__)

        self._instances = {}

        operations_pb2_grpc.add_OperationsServicer_to_server(self, server)

    # --- Public API ---

    def add_instance(self, instance_name, instance):
        """Registers a new servicer instance.

        Args:
            instance_name (str): The new instance's name.
            instance (OperationsInstance): The new instance itself.
        """
        self._instances[instance_name] = instance

    # --- Public API: Servicer ---

    @authorize(AuthContext)
    def GetOperation(self, request, context):
        self.__logger.debug("GetOperation request from [%s]", context.peer())

        try:
            name = request.name

            instance_name = self._parse_instance_name(name)
            instance = self._get_instance(instance_name)

            operation_name = self._parse_operation_name(name)
            operation = instance.get_operation(operation_name)
            op = operations_pb2.Operation()
            op.CopyFrom(operation)
            op.name = name
            return op

        except InvalidArgumentError as e:
            self.__logger.error(e)
            context.set_details(str(e))
            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)

        return operations_pb2.Operation()

    @authorize(AuthContext)
    def ListOperations(self, request, context):
        self.__logger.debug("ListOperations request from [%s]", context.peer())

        try:
            # The request name should be the collection name
            # In our case, this is just the instance_name
            instance_name = request.name
            instance = self._get_instance(instance_name)

            result = instance.list_operations(request.filter,
                                              request.page_size,
                                              request.page_token)

            for operation in result.operations:
                operation.name = "{}/{}".format(instance_name, operation.name)

            return result

        except InvalidArgumentError as e:
            self.__logger.error(e)
            context.set_details(str(e))
            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)

        return operations_pb2.ListOperationsResponse()

    @authorize(AuthContext)
    def DeleteOperation(self, request, context):
        self.__logger.debug("DeleteOperation request from [%s]", context.peer())

        try:
            name = request.name

            instance_name = self._parse_instance_name(name)
            instance = self._get_instance(instance_name)

            operation_name = self._parse_operation_name(name)
            instance.delete_operation(operation_name)

        except InvalidArgumentError as e:
            self.__logger.error(e)
            context.set_details(str(e))
            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)

        return Empty()

    @authorize(AuthContext)
    def CancelOperation(self, request, context):
        self.__logger.debug("CancelOperation request from [%s]", context.peer())

        try:
            name = request.name

            instance_name = self._parse_instance_name(name)
            instance = self._get_instance(instance_name)

            operation_name = self._parse_operation_name(name)
            instance.cancel_operation(operation_name)

        except InvalidArgumentError as e:
            self.__logger.error(e)
            context.set_details(str(e))
            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)

        return Empty()

    # --- Private API ---

    def _parse_instance_name(self, name):
        """ If the instance name is not blank, 'name' will have the form
        {instance_name}/{operation_uuid}. Otherwise, it will just be
        {operation_uuid} """
        names = name.split('/')
        return '/'.join(names[:-1]) if len(names) > 1 else ''

    def _parse_operation_name(self, name):
        names = name.split('/')
        return names[-1] if len(names) > 1 else name

    def _get_instance(self, name):
        try:
            return self._instances[name]

        except KeyError:
            raise InvalidArgumentError("Instance doesn't exist on server: [{}]".format(name))


import sys
from PyQt5.QtWidgets import QApplication, QWidget, QPushButton
from PyQt5.QtWidgets import QLCDNumber, QLabel, QLineEdit
from PyQt5.QtGui import QPixmap

class Example(QWidget):
    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):

        self.setGeometry(300, 300, 300, 400)
        self.setWindowTitle('Img')

        self.btn = QPushButton('Показать', self)
        self.btn.resize(self.btn.sizeHint())
        self.btn.move(100, 80)
        self.btn.clicked.connect(self.hello)

        # _________input1____________________#
        self.name_label = QLabel(self)
        self.name_label.setText("Имя файла:  ")
        self.name_label.move(40, 50)

        self.name_input = QLineEdit(self)
        self.name_input.move(130, 50)

        self.pix_label = QLabel(self)
        self.pix_label.setText("testtesttesttesttesttesttesttest \n srfgwrfgwerf")
        self.pix_label.setGeometry(40, 150, 200, 200)

    def hello(self):
        a = self.name_input.text()
        pixmap = QPixmap(a)
        self.pix_label.setPixmap(pixmap)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = Example()
    ex.show()
    sys.exit(app.exec())


import cv2
import numpy as np
from matplotlib import pyplot as plot

def converted(p):
	p[0]=p[0]/2
	p[1]=p[1]*255/100
	p[2]=p[2]*255/100
	return np.array(p)

def imgNeg(image):
	return 255-image

cap=cv2.VideoCapture(0)
while(1):
	#_,raw=cap.read()
	#daw=cv2.Canny(raw,100,250)	#Intensity gradient-Min to max
	#hsv=cv2.cvtColor(raw, cv2.COLOR_BGR2HSV)

	im = cv2.imread('Pictures/Webcam/2015-05-18-035043.jpg')
	imgray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
	thresh = cv2.threshold(imgray,127,255,0)
	contours= cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
	im = cv2.drawContours(im, contours, -1, (0,255,0), 3)

	#canny=cv2.Canny(hsv,100,250)
	#canny=imgNeg(canny)

	#lower_white=[0,0,65]	#Fine tuned values trained specifically to recognise my tshirt! 145,23,12 also works!
	#upper_white=[360,22,100]
	#lower_blue=[209,30,10]
	#upper_blue=[265,90,97]
	#lower_red=[322,45,5]
	#upper_red=[360,80,90]

	#wmask=cv2.inRange(hsv,converted(lower_white),converted(upper_white))
	#wmask=cv2.medianBlur(wmask,5)
	#wmask=cv2.GaussianBlur(wmask,(5,5),0)

	#bmask=cv2.inRange(hsv,converted(lower_blue),converted(upper_blue))
	#rmask=cv2.inRange(hsv,converted(lower_red),converted(upper_red))
	#rmask=cv2.medianBlur(rmask,5)
	#daw=cv2.Canny(rmask,100,250)
	#whitened_mask=cv2.bitwise_and(raw,raw,mask=wmask)
	#blued_mask=cv2.bitwise_and(raw,raw,mask=bmask)
	#reddened_mask=cv2.bitwise_and(raw,raw,mask=rmask)
	#rmask=cv2.medianBlur(rmask,5)

	#gray=cv2.cvtColor(raw,cv2.COLOR_BGR2GRAY)
	#thresh=cv2.threshold(gray,127,255,0)
	#contours=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
	#raw=cv2.drawContours(raw,contours,-1,(0,255,0),3)

	cv2.imshow('Its such a good vibration!',raw)
	#cv2.imshow('whitened Mask',whitened_mask)
	#cv2.imshow('Canny',daw)
	#cv2.imshow('Reddened Mask',rmask)
	#cv2.imshow('Trackin \'n Crackin\'!',daw)
	#cv2.imshow('Blued Mask',blued_mask)
	#cv2.imshow('My Sketch!',canny)
	k=cv2.waitKey(5) & 0xff
	if k==27:
		break
cv2.destroyAllWindows()


from unittest import mock
from utils.komp2_api import get_allele2gene_map
from utils.models import Allele
import pytest
from sqlalchemy.exc import SQLAlchemyError

@mock.patch('utils.komp2_api.Session')
def test_get_allele2gene_map_one_allele(mock_session):
    test_allele = Allele()
    test_allele.acc = "MGI:4244706"
    test_allele.gf_acc = "MGI:1915571"
    mock_session.return_value.query(Allele).all.return_value = [test_allele]
    allele_gene_map = get_allele2gene_map()
    assert(allele_gene_map["MGI:4244706"] == "MGI:1915571")

@mock.patch('utils.komp2_api.Session')
def test_get_allele2gene_map_no_alleles(mock_session):
    mock_session.return_value.query(Allele).all.return_value = []
    allele_gene_map = get_allele2gene_map()
    assert(allele_gene_map == {})

@mock.patch('utils.komp2_api.Session')
def test_raises(mock_session):
    mock_session.return_value.query(Allele).all.side_effect = lambda: exec('raise(SQLAlchemyError("some info"))')
    with pytest.raises(Exception) as excinfo:
        get_allele2gene_map()
    assert str(excinfo.value) == 'some info'


import matplotlib.pyplot as plt
import pandas as pd
from wordcloud import WordCloud
import ast

def polarity_distribution(df, file_name,title):
    """
    Plot distribution of polarity
    Parameters
    ----------
    df: pd.DataFrame
    file_name: str
    title: str

    Returns
    -------
    None
    """
    save_path = '../reports/figures/'
    if 'sentiment_target' in df.columns:
        plt.figure(figsize=(20,15))
        df_pos = df['sum_polarity_unique_emoji'][df['sentiment_target']=='positive']
        df_neg = df['sum_polarity_unique_emoji'][df['sentiment_target']=='negative']
        plt.hist(df_pos,bins = 20,label='positive')
        plt.hist(df_neg,bins = 20,label='negative')
        plt.title(title,fontsize=25)
        plt.xlabel('Sum of polarity',fontsize=15)
        plt.ylabel('Counts',fontsize=15)
        plt.legend()
    else:
        plt.figure(figsize=(20,15))
        plt.hist(df['sum_polarity_unique_emoji'],bins=40)
        plt.title(title,fontsize=25)
        plt.xlabel('Sum of polarity',fontsize=15)
        plt.ylabel('Counts',fontsize=15)
    plt.savefig(save_path+file_name)

def target_distribution(df,file_name,title):
    """
    Plot target distribution for classes
    Parameters
    ----------
    df: pd.DataFrame
    file_name: str
    title: str

    Returns
    -------
    None
    """
    save_path = '../reports/figures/'
    plt.figure(figsize=(15,10))
    counts = df['sentiment_target'].value_counts()
    plt.bar(counts.index, counts.values)
    plt.title(title,fontsize=25)
    plt.xlabel('Target',fontsize=15)
    plt.ylabel('Counts',fontsize=15)
    plt.savefig(save_path+file_name)

def word_cloud(df,emotion):
    """
    plot word cloud for the most important 100 words
    Parameters
    ----------
    df: pd.DataFrame
    emotion: str

    Returns
    -------

    """
    save_path = '../reports/figures/'
    connect_word_in_tweet = []
    df_neg = df[df['sentiment_target']==emotion]
    for case in df_neg['tokens']:
        x = ast.literal_eval(case)
        x = ' '.join(x)
        connect_word_in_tweet.append(x)
        text = ' '.join(connect_word_in_tweet)
        text = text.replace('-pron-','')
    wordcloud = WordCloud(max_words=100,width=1920, height=1080).generate(text)
    plt.figure( figsize=(20,10) )
    plt.title(emotion)
    plt.imshow(wordcloud, interpolation="bilinear")
    plt.savefig(save_path + 'wordcloud_' + emotion + '.png', facecolor='k', bbox_inches='tight')


# List Comprehension - Output a list from manipulating the values
# from and original list

# [_for_in_]

nums = [1, 3, 5]

[x*10 for x in nums]

# Output
[10, 30, 50]

# List Comprehension vs Looping

nums = [1, 2, 3, 4, 5]
double_nums = []

for num in nums:
    double_num = num * 2
    double_nums.append(double_num)
print(double_nums)

# OR

double_nums = [nums * 2 for num in nums]

# Examples of List Comprehension

name = 'colt'

[char.upper() for char in name]
# Output
['C', 'O', 'L', 'T']

friends = ['jim', 'jessee', 'johnny']

[friends[0].upper() for x in friends]
# Output
['Jim', 'Jessee', 'Johnny']

[num*10 for num in range(1,6)]
# Output
[10, 20, 30, 40, 50]

nums = [1, 3, 5, 7]
num_to_string = [str(x) for x in nums]
print(num_to_string)
# Output
['1', '3', '5', '7']

board = [[x for x in range(1, 4)] for y in range(1, 5)]
# The first part - This creates a single list [1, 2, 3]
# The second part - This does everything in the first part for range(1, 5)

# Output
[[1, 2, 3], [1, 2, 3], [1, 2, 3], [1, 2, 3]]

[["X" if num % 2 != 0 else "O" for num in range(1, 4)] for x in range(1, 4)]
# Working backwards this time...
# We know there will be three lists based on the "for x in rang.."
# The beginning says put an "X" if even or an "O" if odd in range(1, 4)
# or three values in a list

# Output
[['X', 'O', 'X'], ['X', 'O', 'X'], ['X', 'O', 'X']]

users = [
	{"username": "samuel", "tweets": ["I love cake", "I love pie", "hello world!"]},
	{"username": "katie", "tweets": ["I love my cat"]},
	{"username": "jeff", "tweets": []},
	{"username": "bob123", "tweets": []},
	{"username": "doggo_luvr", "tweets": ["dogs are the best", "I'm hungry"]},
	{"username": "guitar_gal", "tweets": []}
]

# NOTE: You don't need to use u['tweets'] == None or len(u['tweets'] == 0) if
# we are only looking for True or False. [] is equal to False anyway!

#extract inactive users using list comprehension:
inactive_users2= [user for user in users if not user["tweets"]]

# extract usernames of inactive users w/ list comprehension
usernames2 = [user["username"].upper() for user in users if not user["tweets"]]


# added aggressive colony, strength of Blobs as indicator of who gets to eat
# when two agro colonies meet, only stronger one gets to eat
# swapped 'if len(list) > 0' to 'if not list'
# agroBlobs now spawn with a slightly mutated strength
# adjusted the scoreboard to show average strength of the colony

import pygame, numpy, random

# Define some colors
BLACK    = (   0,   0,   0)
WHITE    = ( 255, 255, 255)
RED      = ( 255,   0,   0)
GREEN    = (   0, 255,   0)
BLUE     = (   0,   0, 255)
PURPLE   = ( 128,   0, 128)
# Set window parameters
pygame.init()
size = width, height = 680, 680
pixel = 5
pixWidth, pixHeight = width/pixel, height/pixel
pygame.display.set_caption("Game of Life")
screen = pygame.display.set_mode(size)
keys_down = set()
myfont = pygame.font.SysFont("monospace", 25)
# Used to manage how fast the screen updates
clock = pygame.time.Clock()

# Lay seed for random fuctions
random.seed

# Classes

class Blob:
    """Blob is a green or red block that moves around, repopulates, eats and dies"""

    def __init__(self, s, l, a=0):
        self.strength = s
        self.life = l
        self.age = a

    def strength(self):
        return self.strength

class TheMap:
    """Map stores screen size as well as 3D array of Blobs, also includes funtions for interaction of Blobs"""

    def __init__(self,pixWidth,pixHeight,pixel):
        """Initialises the map of given width and height with 500 Blobs of each species"""

        self.pixWidth = pixWidth
        self.pixHeight = pixHeight
        self.pixel = pixel
        self.turn = 0

        self.count = [ 0 for i in range(nColonies) ]
        self.avgStr = [ 0 for i in range(nColonies) ]

        self.array = [ [ [ [] for z in range(nColonies) ] for y in range(pixHeight) ] for x in range(pixWidth) ]
        for i in range(nColonies):
            for n in range(1000):
                x = random.randrange(0,pixWidth)
                y = random.randrange(0,pixHeight)

                s = numpy.random.normal(50,1)       # strength set as a normal distribution
                l = numpy.random.normal(life,1)     # life span altered by plus/minus normal distribution

                self.array[x][y][i].append(Blob(s,l))

    def moveBlobs(self):
        """Moves blobs randomly within provided screen size, also counts each colony size"""

        # --- Create an arrray to store next turn
        tmp_array = [ [ [ [] for z in range(nColonies) ] for y in range(pixHeight) ] for x in range(pixWidth) ]

        # --- Set count to 0
        for i in range(nColonies):
            self.count[i] = 0
            self.avgStr[i] = 0

        # --- Go through each pixel of each layer of Map
        for i in range(nColonies):
            for x in range(self.pixWidth):
                for y in range(self.pixHeight):

                    while self.array[x][y][i]:
                        self.count[i] += 1      # count the blob that is beeing moved
                        self.avgStr[i] += self.array[x][y][i][-1].strength

                        xx = x + random.choice([-1,0,1])
                        yy = y + random.choice([-1,0,1])

                        if xx >= self.pixWidth:
                            xx = 0
                        elif xx < 0:
                            xx = self.pixWidth - 1
                        if yy >= self.pixHeight:
                            yy = 0
                        elif yy < 0:
                            yy = self.pixHeight - 1

                        if len(tmp_array[xx][yy][i]) < 5:
                            tmp_array[xx][yy][i].append(self.array[x][y][i].pop())
                        else:
                            break

            self.avgStr[i] /= self.count[i]
        self.array = tmp_array

    def actBlobs(self):
        """Checks if aggresive blobs ate something and kills the old ones """

        # --- Create an arrray to store next turn
        tmp_array = [ [ [ [] for z in range(nColonies) ] for y in range(pixHeight) ] for x in range(pixWidth) ]

        # --- Go through each pixel in Map
        for x in range(self.pixWidth):
                for y in range(self.pixHeight):

                    # --- Remove the old aggressive Blobs
                    for i in range(1,nColonies):
                        for dying in self.array[x][y][i]:
                            dying.age += 1
                            if dying.age > life:
                                self.array[x][y][i].remove(dying)

                    # --- Resolve conflicts between aggressive Blobs
                    winner = 2
                    if self.array[x][y][1]:
                        winner = 1
                        if self.array[x][y][2]:
                            strength = [0,0]
                            for i in range(nColonies-1):
                                for blob in self.array[x][y][i+1]:
                                    strength[i] += blob.strength
                                strength[i] /= len(self.array[x][y][i+1])
                            if strength[1] == strength[0]:
                                winner = random.choice([1,2])
                            elif strength[1] > strength[0]:
                                winner = 2

                    # --- Sort the agroBlobs by strenght (increasing)
                    for i in range(1,nColonies):
                        self.array[x][y][i].sort(key=lambda Blob: Blob.strength) # sorts list with strenght as a key

                    # --- Let the winner eat green Blobs
                    while self.array[x][y][winner] and self.array[x][y][0]:
                        self.array[x][y][0].pop()
                        s = self.array[x][y][winner][-1].strength + numpy.random.normal(0,2)
                        tmp_array[x][y][winner].append(self.array[x][y][winner].pop())

                        xx = x + random.choice([-1,0,1])
                        yy = y + random.choice([-1,0,1])

                        if xx >= self.pixWidth:
                            xx = 0
                        elif xx < 0:
                            xx = self.pixWidth - 1
                        if yy >= self.pixHeight:
                            yy = 0
                        elif yy < 0:
                            yy = self.pixHeight - 1

                        if len(tmp_array[xx][yy][winner]) < 5:
                            tmp_array[xx][yy][winner].append(Blob(s,life))
                        else:
                            break

                    # --- Surviving Greens get a chance to repopulate
                    while self.array[x][y][0]:

                        if random.random() < span_chance:
                            xx = x + random.choice([-1,0,1])
                            yy = y + random.choice([-1,0,1])

                            if xx >= self.pixWidth:
                                xx = 0
                            elif xx < 0:
                                xx = self.pixWidth - 1
                            if yy >= self.pixHeight:
                                yy = 0
                            elif yy < 0:
                                yy = self.pixHeight - 1

                            if len(tmp_array[xx][yy][0]) < 5:
                                tmp_array[xx][yy][0].append(self.array[x][y][0][-1])
                            else:
                                break

                        if len(tmp_array[x][y][0]) < 5:
                            tmp_array[x][y][0].append(self.array[x][y][0].pop())
                        else:
                            break

                    # --- Transfer the remaining agroBlobs to new array
                    for i in range(1,nColonies):
                        while self.array[x][y][i]:
                            if len(tmp_array[x][y][i]) < 5:
                                tmp_array[x][y][i].append(self.array[x][y][i].pop())
                            else:
                                break

        # --- Update the array
        self.array = tmp_array
        self.turn += 1

    def draw(self,screen):
        """Draws the map with blobs"""

        for x in range(self.pixWidth):
            for y in range(self.pixHeight):
                xy = [x*self.pixel,y*self.pixel,self.pixel,self.pixel]

                if self.array[x][y][1]:
                    pygame.draw.rect(screen,RED,xy)
                    if self.array[x][y][2]:
                        pygame.draw.polygon(screen,PURPLE,[[x*self.pixel,y*self.pixel],[(x+1)*self.pixel,y*self.pixel],[x*self.pixel,(y+1)*self.pixel]])
                elif self.array[x][y][2]:
                        pygame.draw.rect(screen,PURPLE,xy)
                elif self.array[x][y][0]:
                    pygame.draw.rect(screen,GREEN,xy)

        pygame.draw.rect(screen,BLACK,[0,0,200,85])
        pygame.draw.rect(screen,WHITE,[0,0,200,85],2)

        label = myfont.render('Size', 1, WHITE)
        screen.blit(label, (3,  1))
        label = myfont.render(str(self.count[0]), 1, GREEN)
        screen.blit(label, (3, 21))
        label = myfont.render(str(self.count[1]), 1, RED)
        screen.blit(label, (3, 41))
        label = myfont.render(str(self.count[2]), 1, PURPLE)
        screen.blit(label, (3, 61))

        label = myfont.render('avgStr', 1, WHITE)
        screen.blit(label, (80,  1))
        #label = myfont.render(str(self.avgStr[0]), 1, GREEN)
        #screen.blit(label, (80, 21))
        label = myfont.render("{:.3f}".format(self.avgStr[1]), 1, RED)
        screen.blit(label, (80, 41))
        label = myfont.render("{:.3f}".format(self.avgStr[2]), 1, PURPLE)
        screen.blit(label, (80, 61))
# Functions

# Variables
nColonies = 3

life = 12		        # how many turns it takes for aggresive blob to die
span_chance = 0.20		# percentage chance of green blob spanning offspring

done = False
RUNNING, PAUSE = 0,1
state = RUNNING

# Pre-start set up
theMap = TheMap(pixWidth,pixHeight,pixel)     # initalises theMap with blobs
theMap.draw(screen)

# -------- Main Program Loop -----------
while not done:

    # --- Event loop (not yet fully useful)

    for event in pygame.event.get():        # did user do something
        if event.type == pygame.KEYDOWN:    # if user pushed any key
            if event.key == pygame.K_SPACE: state = not state
            if event.key == pygame.K_ESCAPE:
                done = True
                state = PAUSE

    if state == RUNNING:
        # --- Game logic
        theMap.moveBlobs()  # moves the Blobs around in the array

        theMap.actBlobs()  # aggressive Blobs eat Green ones, they all reproduce

        # --- Clear the screen
        screen.fill(BLACK)

        # --- Drawing new screen
        theMap.draw(screen)

        # --- Update screen with what was drawn
        pygame.display.flip()

        # --- Limits frames per second
        clock.tick(10)


from __future__ import division

import numpy as np
import operator
import scipy.sparse as sp
from numpy.testing import assert_raises
from pyoperators import SparseOperator
from pyoperators.utils.testing import assert_same

A = np.array([[1, 0, 2, 0],
              [0, 0, 3, 0],
              [4, 5, 6, 0],
              [1, 0, 0, 1]])
vecs = [[1, 0, 0, 0],
        [0, 1, 0, 0],
        [0, 0, 1, 0],
        [0, 0, 0, 1]]
formats = 'bsr,csr,csc,coo,dia,dok'.split(',')

def test_matvec():
    def func(format):
        cls = getattr(sp, format + '_matrix')
        so = SparseOperator(cls(A))
        for vec in vecs:
            assert_same(so(vec), np.dot(A, vec))
            assert_same(so.T(vec), np.dot(A.T, vec))
        assert_same(so.todense(), A)
        assert_same(so.T.todense(), A.T)
    for format in formats:
        yield func, format

def test_shapes():
    def func(format):
        cls = getattr(sp, format + '_matrix')
        shapein = (2, 2)
        shapeout = (1, 4, 1)
        so = SparseOperator(cls(A), shapein=shapein, shapeout=shapeout)
        for vec in vecs:
            assert_same(so(np.reshape(vec, shapein)),
                        np.dot(A, vec).reshape(shapeout))
            assert_same(so.T(np.reshape(vec, shapeout)),
                        np.dot(A.T, vec).reshape(shapein))
    for format in formats:
        yield func, format

def test_update_output():
    def func(format):
        cls = getattr(sp, format + '_matrix')
        so = SparseOperator(cls(A))
        out = np.zeros(4, dtype=int)
        outT = np.zeros(4, dtype=int)
        for vec in vecs:
            so(vec, out, operation=operator.iadd)
            so.T(vec, outT, operation=operator.iadd)
        assert_same(out, np.sum(A, axis=1))
        assert_same(outT, np.sum(A, axis=0))
    for format in formats:
        yield func, format

def test_error1():
    values = (sp.lil_matrix(A), np.zeros((10, 4)),
              np.matrix(np.zeros((10, 4))), 3)

    def func(v):
        assert_raises(TypeError, SparseOperator, v)
    for v in values:
        yield func, v

def test_error2():
    def func(format):
        cls = getattr(sp, format + '_matrix')
        sm = cls(A)
        shapein = (2, 3)
        shapeout = (1, 4, 2)
        assert_raises(ValueError, SparseOperator, sm, shapein=shapein)
        assert_raises(ValueError, SparseOperator, sm, shapeout=shapeout)
    for format in formats:
        yield func, format

def test_error3():
    def func(format):
        cls = getattr(sp, format + '_matrix')
        sm = cls(A)
        so = SparseOperator(sm)
        out = np.zeros(4, dtype=int)
        assert_raises(ValueError, so, vecs[0], out, operation=operator.imul)
    for format in formats:
        yield func, format


import json
import dataClasses

def readRaw(filename):
    infile = open(filename,"r")
    indata = infile.read()
    infile.close()
    return indata

def makeEvents(rawdata):
    ou = []
    for x in rawdata["EVENTS"]:
        #x is an event structure
        ev = dataClasses.data_event()
        ev.fromDict(x)
        ou.append(ev)
    return ou

def read(filename):
    raw = readRaw(filename)
    st = json.loads(raw)
    del(raw)
    osc_meta = st["OSC_META"]
    ev_list = makeEvents(st)
    del(st)
    return [osc_meta,ev_list]


import numpy as np
import matplotlib.pyplot as plt

from numpy.fft import fft, fftfreq, ifft

lx = 100 # time in seconds
n = 1000 # number of points

x = np.linspace(1, lx, n) # Time

omg = 2.0*np.pi/lx

y1 = 3.0*np.cos(5.0*omg*x)
y2 = 2.0*np.sin(10.0*omg*x)
y3 = 1.0*np.sin(20.0*omg*x)

y = y3+y2+y1

plt.figure(1)
plt.plot(x, y, '-b')
# plt.show()

ffts = fftfreq(n)
fftvals = fft(y)

#further processing
mask = ffts > 0
ffttheo = 2.0*abs(fftvals/n)

plt.figure(2)
plt.plot(ffts, fftvals, '-g')
# plt.plot(ffts[mask], ffttheo[mask], '-g')
plt.show()


import sys
from os.path import abspath, join

import numpy as np
import scipy.optimize as so
import scipy.spatial as sp
import trimesh
from PySide2.QtWidgets import QApplication
from PySide2.QtWidgets import QMainWindow
from PySide2.QtWidgets import QMessageBox

from loader.pose_loader import PoseLoader
from loader.regressor_loader import RegressorLoader
from loader.smpl_loader import SmplLoader
from main_window import Ui_MainWindow

NEUTRAL = 'n'
FEMALE = 'f'
MALE = 'm'

models = {
    NEUTRAL: 'basic_model_neutral.pkl',
    # FEMALE: 'basic_model_female.pkl',
    # MALE: 'basic_model_male.pkl',
}

class MainWindow(QMainWindow, Ui_MainWindow):

    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent=parent)
        self.setupUi(self)

        self.scene_chache = {}
        self.pose_loader = PoseLoader()
        self.smpl_loader = SmplLoader()
        self.regressor_loader = RegressorLoader()

        self.neutral_button.toggled.connect(lambda: self._init_widget())
        if FEMALE not in models:
            self.female_button.setVisible(False)
            self.female_button.toggled.connect(lambda: self._init_widget())
        if MALE not in models:
            self.male_button.setVisible(False)
            self.male_button.toggled.connect(lambda: self._init_widget())

        self.poses_box.currentIndexChanged.connect(lambda: self._init_widget())

        self.regressor_name.textChanged.connect(lambda: self.check_convert_button())
        self.convert_button.clicked.connect(lambda: self.convert_scenes_to_regressor())

        self.reset_button.clicked.connect(lambda: self.reset())

        self._init_poses()
        self._init_regressors()
        self._init_widget()

    def _init_poses(self):
        poses_path = abspath(join(__file__, '..', 'smpl', 'poses', 'cmu_smpl_01_01.pkl'))
        self.pose_loader.init_poses(poses_path)
        poses, shapes, transforms = self.pose_loader.sample_poses()

        for gender, file_name in models.items():
            smpl_model_path = abspath(join(__file__, '..', 'smpl', 'models', file_name))
            self.smpl_loader.init_model(smpl_model_path)

            for i, (pose, shape, transform) in enumerate(zip(poses, shapes, transforms)):
                verts = self.smpl_loader.load_vertices(pose, shape, transform)
                faces = self.smpl_loader.faces

                mesh = trimesh.Trimesh(vertices=verts,
                                       faces=faces,
                                       vertex_colors=[200, 200, 200, 255],
                                       face_colors=[0, 0, 0, 0],
                                       use_embree=False,
                                       process=False)

                transform = trimesh.transformations.rotation_matrix(np.deg2rad(-90), [1, 0, 0], mesh.centroid)
                mesh.apply_transform(transform)

                transform = trimesh.transformations.rotation_matrix(np.deg2rad(-90), [0, 1, 0], mesh.centroid)
                mesh.apply_transform(transform)

                key = gender + '_pose_' + str(i)
                self.scene_chache[key] = mesh.scene()

        for i in range(poses.shape[0]):
            self.poses_box.addItem('Pose ' + str(i))

    def _init_regressors(self):
        regressors_path = abspath(join(__file__, '..', 'regressors', '*.npy'))
        self.regressor_loader.init_regressors(regressors_path)

    def _init_widget(self):
        gender = self.get_checked_gender()
        index = self.poses_box.currentIndex()
        index = index if index >= 0 else 0
        key = gender + '_pose_' + str(index)
        scene = self.scene_chache[key]
        self.openGLWidget.initialize_scene(scene, self.smpl_loader.j_regressor, self.regressor_loader.joint_regressor)
        self.openGLWidget.updateGL()

    def get_checked_gender(self):
        if self.neutral_button.isChecked():
            gender = NEUTRAL
        elif self.female_button.isChecked():
            gender = FEMALE
        elif self.male_button.isChecked():
            gender = MALE
        else:
            raise Exception('no button checked')

        return gender

    def check_convert_button(self):
        enable_button = self.regressor_name.text() != ''
        self.convert_button.setEnabled(enable_button)

    def convert_scenes_to_regressor(self):
        self.brute_force_closest_vertex_to_joints()

    def brute_force_closest_vertex_to_joints(self):
        scenes = self.get_valid_scenes()

        # calculate 50 nearest vertex ids for all meshes and joints
        closest_k = 50
        candidates = {}
        keep_searching = True
        while keep_searching:
            for key, scene in scenes.items():
                joint = np.squeeze(scene['joint'].vertices)
                distances, vertex_ids = scene['mesh'].kdtree.query(joint, closest_k)
                candidates[key] = {vertex_id: dist for vertex_id, dist in zip(vertex_ids, distances)}

            # only keep common ids
            from functools import reduce
            common_ids = reduce(np.intersect1d, [list(c.keys()) for c in candidates.values()])

            # calculate average distance per mesh/joint for valid ids
            mean_dist = [np.mean([c[common_id] for c in candidates.values()]) for common_id in common_ids]
            mean_dist = {common_id: dist for common_id, dist in zip(common_ids, mean_dist)}
            mean_dist = {k: v for k, v in sorted(mean_dist.items(), key=lambda item: item[1])}

            if len(list(mean_dist)) == 0:
                closest_k += 10
            else:
                keep_searching = False

        # pick closest vertex with min average distance to all joints per mesh
        closest_id = list(mean_dist)[0]
        final_vertices = [closest_id]
        mean_dist.pop(closest_id)

        while len(final_vertices) < 10:
            # calculate all distance combinations between valid vertices
            vertex_ids = list(mean_dist)
            id_dist = [sp.distance.cdist(s['mesh'].vertices[final_vertices], s['mesh'].vertices[vertex_ids]) for s in
                       scenes.values()]
            id_dist = np.mean(id_dist, axis=0)

            # min the ratio between distances to joint and distance to all other vertices
            best_dist = list(mean_dist.values()) / id_dist
            best_id = np.argmin(best_dist)

            # max the difference between distance to all other vertices and distances to joint
            best_dist = id_dist - list(mean_dist.values())
            best_id = np.argmax(best_dist)

            n, m = np.unravel_index(best_id, best_dist.shape)
            best_id = vertex_ids[m]

            final_vertices.append(best_id)
            mean_dist.pop(best_id)

        vertices, joints = [], []
        for scene in scenes.values():
            verts = np.asarray(scene['mesh'].vertices).reshape([-1, 3])
            verts = verts[final_vertices]
            vertices.append(verts)
            joint = np.asarray(scene['joint'].vertices).reshape([-1, 3])
            joints.append(joint)

        vertices = np.stack(vertices).transpose([0, 2, 1]).reshape([-1, len(final_vertices)])
        joints = np.stack(joints).transpose([0, 2, 1]).reshape([-1])

        vertex_weight = np.zeros([6890, ])
        weights = so.nnls(vertices, joints)[0]
        vertex_weight[final_vertices] = weights

        file = join('regressors', 'regressor_{}.npy'.format(self.regressor_name.text()))
        with open(file, 'wb') as f:
            vertex_weight = vertex_weight.astype(np.float32)
            vertex_weight = np.expand_dims(vertex_weight, -1)
            np.save(f, vertex_weight)

            widget = QMessageBox(
                icon=QMessageBox.Information,
                text='Regressor file successfully saved to: {}\n\nClick Reset to start again'.format(file),
                buttons=[QMessageBox.Ok]
            )
            widget.exec_()

        vertex_weight = np.squeeze(vertex_weight)
        self.convert_button.setEnabled(False)
        self.regressor_name.setEnabled(False)

        for scene in self.scene_chache.values():
            mesh = scene.geometry['geometry_0']
            mesh.visual.vertex_colors = [200, 200, 200, 255]
            mesh.visual.vertex_colors[final_vertices] = [0, 255, 0, 255]

            x = np.matmul(vertex_weight, mesh.vertices[:, 0])
            y = np.matmul(vertex_weight, mesh.vertices[:, 1])
            z = np.matmul(vertex_weight, mesh.vertices[:, 2])
            joints = np.vstack((x, y, z)).T
            joints = trimesh.PointCloud(joints, colors=[0, 255, 0, 255])
            scene.add_geometry(joints, geom_name='new_joints')

    def get_valid_scenes(self):
        valid_scenes = {}

        scenes, scales, centroids = [], [], []
        for scene in self.scene_chache.values():
            if 'joint_0' not in scene.geometry:
                continue
            scenes.append(scene)
            scales.append(scene.scale)
            centroids.append(scene.centroid)

        for i, scene in enumerate(scenes):
            valid_scenes[i] = {'mesh': scene.geometry['geometry_0'], 'joint': scene.geometry['joint_0']}

        return valid_scenes

    def reset(self):
        self.regressor_name.setText('')
        self.regressor_name.setEnabled(True)
        self.poses_box.clear()

        self._init_poses()
        self._init_regressors()
        self._init_widget()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    window = MainWindow()
    window.show()

    try:  # main loop
        sys.exit(app.exec_())
    except SystemExit:
        pass


from __future__ import with_statement, division
import xml.etree.ElementTree as et
from xml.dom import minidom
import struct
import array
import sys
import re
import os
from imctools.io.imcacquisitionbase import ImcAcquisitionBase
from imctools.io.abstractparserbase import AbstractParserBase
from imctools.io.mcdxmlparser import McdXmlParser
import imctools.io.mcdxmlparser as mcdmeta
from imctools.io.abstractparserbase import AcquisitionError

from collections import defaultdict

"""
Main class
"""

class McdParserBase(AbstractParserBase):
    """Parsing data from Fluidigm MCD files

    The McdParser object should be closed using the close
    :param object:
    :return:
    """

    def __init__(self, filename, filehandle = None, metafilename=None):
        """

        :param filename:
        """
        AbstractParserBase.__init__(self)

        if filehandle is None:
            self._fh = open(filename, mode='rb')
        else:
            self._fh = filehandle

        if metafilename is None:
            self._metafh = self._fh
        else:
            self._metafh = open(metafilename, mode='rb')
        self._xml = None
        self._ns = None
        self._acquisition_dict = None
        self.retrieve_mcd_xml()
        self.parse_mcd_xml()

    @property
    def filename(self):
        "Return the name of the open file"
        return self._fh.name

    @property
    def n_acquisitions(self):
        """
        Number of acquisitions in the file
        :return:
        """
        return len(self.meta.get_acquisitions())

    @property
    def acquisition_ids(self):
        """
        Acquisition IDs
        :return:
        """
        return list(self.meta.get_acquisitions().keys())

    def get_acquisition_description(self, ac_id, default=None):
        """
        Get the description field of the acquisition
        :param ac_id:
        :return:
        """
        acmeta = self.meta.get_acquisition_meta(ac_id)
        desc = acmeta.get(mcdmeta.DESCRIPTION, default)
        return desc

    def get_acquisition_buffer(self, ac_id):
        """
        Returns the raw buffer for the acquisition
        :param ac_id: the acquisition id
        :return: the acquisition buffer
        """
        f = self._fh
        ac = self.meta.get_acquisitions()[ac_id]
        data_offset_start = ac.data_offset_start
        data_offset_end = ac.data_offset_end
        data_size = ac.data_size
        n_rows = ac.data_nrows
        n_channel = ac.n_channels
        f.seek(data_offset_start)
        buffer = f.read(data_size)
        return buffer

    def get_acquisition_rawdata(self, ac_id):
        """
        Get the acquisition XML of the acquisition with the id
        :param ac_id: the acquisition id
        :return: the acquisition XML
        """
        f = self._fh

        ac = self.meta.get_acquisitions()[ac_id]
        data_offset_start = ac.data_offset_start
        data_offset_end = ac.data_offset_end
        data_size = ac.data_size
        n_rows = ac.data_nrows
        n_channel = ac.n_channels
        if n_rows == 0:
            raise AcquisitionError('Acquisition ' + ac_id + ' emtpy!')

        f.seek(data_offset_start)
        dat = array.array('f')
        dat.fromfile(f, (n_rows * n_channel))
        if sys.byteorder != 'little':
            dat.byteswap()
        data = [dat[(row * n_channel):((row * n_channel) + n_channel)]
                for row in range(n_rows)]
        return data

    def _inject_imc_datafile(self, filename):
        """
        This function is used in cases where the MCD file is corrupted (missing MCD schema)
        but there is a MCD schema file available. In this case the .schema file can
        be loaded with the mcdparser and then the corrupted mcd-data file loaded
        using this function. This will replace the mcd file data in the backend (containing only
        the schema data) with the real mcd file (not containing the mcd xml).
        """
        self.close()
        self._fh = open(filename, mode='rb')

    def get_nchannels_acquisition(self, ac_id):
        """
        Get the number of channels in an acquisition
        :param ac_id:
        :return:
        """
        ac = self.meta.get_acquisitions()[ac_id]
        return ac.n_channels

    def get_acquisition_channels(self, ac_id):
        """
        Returns a dict with the channel metadata
        :param ac_id: acquisition ID
        :return: dict with key: channel_nr, value: (channel_name, channel_label)
        """
        ac = self.meta.get_acquisitions()[ac_id]
        channel_dict = ac.get_channel_orderdict()
        return channel_dict

    def parse_mcd_xml(self):
        """
        Parse the mcd xml into a metadata object
        """
        self._meta = McdXmlParser(self.xml)

    @property
    def meta(self):
        return self._meta

    @property
    def xml(self):
        return self._xml

    def retrieve_mcd_xml(self, start_str='<MCDSchema', stop_str='</MCDSchema>'):
        """
        Finds the MCD metadata XML in the binary.
        As suggested in the specifications the file is parsed from the end.

        :param fn:
        :param start_str:
        :param stop_str:
        """
        f = self._metafh
        xml_start = self._reverse_find_in_buffer(f, start_str.encode('utf-8'))

        if xml_start == -1:
            start_str = self._add_nullbytes(start_str)
            xml_start = self._reverse_find_in_buffer(f, start_str.encode('utf-8'))

        if xml_start == -1:
            raise ValueError('Invalid MCD: MCD xml start tag not found in file %s' % self.filename)
        else:
            xml_stop = self._reverse_find_in_buffer(f, stop_str.encode('utf-8'))
            if xml_stop == -1:
                stop_str = self._add_nullbytes(stop_str)
                xml_stop = self._reverse_find_in_buffer(f, stop_str.encode('utf-8'))
                # xmls = [mm[start:end] for start, end in zip(xml_starts, xml_stops)]

        if xml_stop == -1:
            raise ValueError('Invalid MCD: MCD xml stop tag not found in file %s' % self.filename)
        else:
            xml_stop += len(stop_str)

        f.seek(xml_start)
        xml = f.read(xml_stop-xml_start).decode('utf-8')
        xml = xml.replace('\x00','')
        #print(xml)
        self._xml = et.fromstring(xml)
        self._ns = '{'+self._xml.tag.split('}')[0].strip('{')+'}'

    def get_imc_acquisition(self, ac_id, ac_description=None):

        data = self.get_acquisition_rawdata(ac_id)
        nchan = self.get_nchannels_acquisition(ac_id)
        channels = self.get_acquisition_channels(ac_id)
        channel_name, channel_label = zip(*[channels[i] for i in range(nchan)])

        img = self._reshape_long_2_cyx(data, is_sorted=True)

        if ac_description is None:
           ac_description = self.meta.get_object(mcdmeta.ACQUISITION, ac_id).metaname

        return ImcAcquisitionBase(image_ID=ac_id,
                                  original_file=self.filename,
                                  data=img,
                                  channel_metal=channel_name,
                                  channel_labels=channel_label,
                                  original_metadata=str(
                                                     et.tostring(self._xml)),
                                  image_description=ac_description,
                                  origin='mcd',
                                  offset=3)

    def save_panorama(self, pid, out_folder, fn_out=None):
        """
        Save all the panorma images of the acquisition
        :param out_folder: the output folder
        """
        pano_postfix = 'pano'
        image_offestfix = 161
        p = self.meta.get_object(mcdmeta.PANORAMA, pid)
        img_start = int(p.properties.get(mcdmeta.IMAGESTARTOFFSET,0)) + image_offestfix
        img_end = int(p.properties.get(mcdmeta.IMAGEENDOFFSET,0)) + image_offestfix

        if img_start-img_end == 0:
            return(0)

        file_end = p.properties.get(mcdmeta.IMAGEFORMAT, '.png').lower()

        if fn_out is None:
            fn_out = p.metaname

        if not(fn_out.endswith(file_end)):
            fn_out += '_'+pano_postfix+'.' + file_end

        buf = self._get_buffer(img_start, img_end)
        with open(os.path.join(out_folder, fn_out), 'wb') as f:
            f.write(buf)

    def save_slideimage(self, sid, out_folder, fn_out=None):
        image_offestfix = 161
        slide_postfix = 'slide'
        default_format = '.png'

        s = self.meta.get_object(mcdmeta.SLIDE, sid)
        img_start = int(s.properties.get(mcdmeta.IMAGESTARTOFFSET, 0)) + image_offestfix
        img_end = int(s.properties.get(mcdmeta.IMAGEENDOFFSET, 0)) + image_offestfix
        slide_format = s.properties.get(mcdmeta.IMAGEFILE, default_format)
        if (slide_format in [None, '', '""', "''"]):
            slide_format = default_format

        slide_format = os.path.splitext(slide_format.lower())
        if slide_format[1] == '':
            slide_format = slide_format[0]
        else:
            slide_format = slide_format[1]

        if img_start-img_end == 0:
            return(0)

        if fn_out is None:
            fn_out = s.metaname
        if not(fn_out.endswith(slide_format)):
            fn_out += '_'+slide_postfix+slide_format

        buf = self._get_buffer(img_start, img_end)
        with open(os.path.join(out_folder, fn_out), 'wb') as f:
            f.write(buf)

    def save_acquisition_bfimage_before(self, ac_id, out_folder, fn_out=None):
        ac_postfix = 'before'
        start_offkey = mcdmeta.BEFOREABLATIONIMAGESTARTOFFSET
        end_offkey = mcdmeta.BEFOREABLATIONIMAGEENDOFFSET

        self._save_acquisition_bfimage(ac_id, out_folder, ac_postfix,
                                       start_offkey, end_offkey, fn_out)

    def save_acquisition_bfimage_after(self, ac_id, out_folder, fn_out=None):
        ac_postfix = 'after'
        start_offkey = mcdmeta.AFTERABLATIONIMAGESTARTOFFSET
        end_offkey = mcdmeta.AFTERABLATIONIMAGEENDOFFSET

        self._save_acquisition_bfimage(ac_id, out_folder, ac_postfix,
                                       start_offkey, end_offkey, fn_out)

    def _save_acquisition_bfimage(self, ac_id, out_folder,
                                 ac_postfix, start_offkey, end_offkey, fn_out=None):
        image_offestfix = 161
        image_format = '.png'
        a = self.meta.get_object(mcdmeta.ACQUISITION, ac_id)
        img_start = int(a.properties.get(start_offkey,0)) + image_offestfix
        img_end = int(a.properties.get(end_offkey,0)) + image_offestfix
        if img_end-img_start == 0:
            return(0)

        if fn_out is None:
            fn_out = a.metaname
        buf = self._get_buffer(img_start, img_end)
        if not(fn_out.endswith(image_format)):
            fn_out += '_'+ac_postfix+ image_format
        with open(os.path.join(out_folder, fn_out), 'wb') as f:
            f.write(buf)

    def save_meta_xml(self, out_folder):
        self.meta.save_meta_xml(out_folder)

    def get_all_imcacquistions(self):
        """
        Returns a list of all nonempty imc_acquisitions
        """
        imc_acs = list()
        for ac in self.acquisition_ids:
            try:
                imcac = self.get_imc_acquisition(ac)
                imc_acs.append(imcac)
            except AcquisitionError:
                pass
            except:
                print('error in acqisition: '+ac)
        return imc_acs

    def _get_buffer(self, start, stop):
        f = self._fh
        f.seek(start)
        buf = f.read(stop-start)
        return buf

    def close(self):
        """Close the file handle."""
        self._fh.close()
        try:
            self._metafh.close()
        except:
            pass

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.close()

    @staticmethod
    def _add_nullbytes(buffer_str):
        """
        Adds nullbits between letters.

        :param buffer_str:
        :return: string with nullbits

        >>> McdParserBase._add_nullbytes('abc')
        'a\\x00b\\x00c\\x00'
        """
        pad_str = ''
        for s in buffer_str:
            pad_str += s + '\x00'
        return pad_str

    @staticmethod
    def _reverse_find_in_buffer(f, s, buffer_size=4096):
        # based on http://stackoverflow.com/questions/3893885/cheap-way-to-search-a-large-text-file-for-a-string
        f.seek(0, 2)

        buf = None
        overlap = len(s) - 1
        bsize = buffer_size +overlap+1
        cur_pos = f.tell() - bsize+1
        offset =  (-2*bsize+overlap)
        first_start=True
        while cur_pos >= 0:
            f.seek(cur_pos)
            buf = f.read(bsize)
            if buf:
                pos = buf.find(s)
                if pos >= 0:
                    return f.tell() - (len(buf) - pos)

            cur_pos = f.tell() +offset
            if (cur_pos < 0) and first_start:
                first_start=False
                cur_pos=0
        return -1