import numpy as np
import random as rnd
import matplotlib.pyplot as plt
import math

xx=np.arange(0,100)
for i in range(99):
    xx=np.append(xx,np.arange(0,100))
yy=np.repeat(np.arange(0,100),100)

food=[]
for i in range(50):
    food=np.append(food,np.random.choice(np.arange(1,11),size=10))
    food=np.append(food,np.zeros(10))
    food=food.astype(int)
food=np.append(food,np.zeros(1000))
food=food.astype(int)
for i in range(50):
    food=np.append(food,np.zeros(10))
    food=food.astype(int)
    food=np.append(food,np.random.choice(np.arange(1,11),size=10))
food=np.append(food,np.zeros(1000))
food=food.astype(int)
for i in range(50):
    food=np.append(food,np.random.choice(np.arange(1,11),size=10))
    food=np.append(food,np.zeros(10))
    food=food.astype(int)
food=np.append(food,np.zeros(1000))
food=food.astype(int)
for i in range(50):
    food=np.append(food,np.zeros(10))
    food=food.astype(int)
    food=np.append(food,np.random.choice(np.arange(1,11),size=10))
food=np.append(food,np.zeros(1000))
food=food.astype(int)
for i in range(50):
    food=np.append(food,np.random.choice(np.arange(1,11),size=10))
    food=np.append(food,np.zeros(10))
    food=food.astype(int)
food=np.append(food,np.zeros(1000))
food=food.astype(int)

males=[[],[]]
for i in range(9998):
    males.append([])
females=[[],[]]
for i in range(9998):
    females.append([])

grid=np.zeros((10000,4))
grid[0:10000,0]=np.ravel(xx)
grid[0:10000,1]=np.ravel(yy)
grid[0:10000,2]=np.ravel(food)

def grow_food():
    for i in range(0,10000):
        N=np.round((100*grid[i,2]*math.exp(0.8))/(100+(grid[i,2]*math.expm1(0.8))))
        grid[i,2]=N

def decay_pheromones():
    for i in range(0,10000):
        A=grid[i,3]*math.exp(-1)
        grid[i,3]=A

def list_sperm():
    for i in range(0,10000):
        if len(males[i])>0 and len(females[i])>0:
            for j in females[i]:
                for k in males[i]:
                    P1.pop[j].sperm.append(P1.pop[k].dauer_1)
                    P1.pop[j].sperm.append(P1.pop[k].dauer_2)

class Worm:
    species="C. nigoni"
    def __init__(self,name):
        self.name=name
        self.gender=rnd.choice(["hermaphrodite","male","female"])
        self.age=0
        self.stage="egg"
        self.location=[rnd.choice(xx),rnd.choice(yy)]
        self.energy=10
        self.L1_time=0
        self.L2d_time=0
        self.dauer_1=rnd.choice(np.arange(15,25))
        self.dauer_2=rnd.choice(np.arange(15,25))
        self.sperm=[]
    @property
    def position(self):
        return((100*self.location[1])+self.location[0])
    @property
    def dauer(self):
        return((self.dauer_1+self.dauer_2)/2)
    def move(self):
        if self.energy>0 or self.stage=="dauer":
            if self.stage!="dauer":
                self.energy=self.energy-0.5
            neighbors=[[(self.location[0]+1)%100,self.location[1]],[(self.location[0]-1)%100,self.location[1]],[self.location[0],(self.location[1]+1)%100],[self.location[0],(self.location[1]-1)%100]]
            list=[]
            for i in range(4):
                list.append(grid[100*neighbors[i][1]+neighbors[i][0],3])
            Sum=(1/(1+list[0]))+(1/(1+list[1]))+(1/(1+list[2]))+(1/(1+list[3]))
            P_0=(1/(1+list[0]))/Sum
            P_1=(1/(1+list[1]))/Sum
            P_2=(1/(1+list[2]))/Sum
            P_3=(1/(1+list[3]))/Sum
            if self.stage=="adult" and self.gender=="male":
                males[self.position].remove(self.name)
            if self.stage=="adult" and self.gender=="female":
                females[self.position].remove(self.name)
            index=np.random.choice([0,1,2,3],p=[P_0,P_1,P_2,P_3])
            self.location=neighbors[index]
            grid[self.position,3]=grid[self.position,3]+1
            if self.stage=="adult" and self.gender=="male":
                males[self.position].append(self.name)
            if self.stage=="adult" and self.gender=="female":
                females[self.position].append(self.name)
            print("The location of",self.name,"is",self.location)
            print("The amount of food is",grid[self.position,2].astype(int))
            print("The amount of pheromones is",grid[self.position,3])
        else:
            if self.stage=="adult" and self.gender=="male":
                males[self.position].remove(self.name)
            if self.stage=="adult" and self.gender=="female":
                females[self.position].remove(self.name)
            print("Worm",self.name,"has died of starvation")
            P1.pop[self.name]="dead starving worm"
    def eat(self):
        print("The location of",self.name,"is the same")
        if self.stage=="L1":
            portion=1
        if self.stage=="L2" or self.stage=="L2d":
            portion=2
        if self.stage=="L3":
            portion=4
        if self.stage=="dauer":
            portion=min(4,grid[self.position,2])
        if self.stage=="L4":
            portion=8
        if self.stage=="adult":
            portion=16
        if self.stage=="old":
            portion=8
        number=0
        while number<portion:
            if grid[self.position,2]==0:
                break
            if self.age==400:
                break
            if self.stage!="dauer":
                grid[self.position,2]=grid[self.position,2]-1
                self.energy=self.energy+1
            self.age=self.age+1
            self.grow()
            number=number+1
        if self.age<400:
            grid[self.position,3]=grid[self.position,3]+1
            print("The amount of food is",grid[self.position,2].astype(int))
            print("The amount of pheromones is",grid[self.position,3])
    def grow(self):
        if self.stage!="dauer":
            self.energy=self.energy-0.5
        if self.age==10:
            self.stage="L1"
            grid[self.position,3]=grid[self.position,3]+1
            print("Worm",self.name,"has reached L1")
        if self.age==20:
            prob=1/(1+math.exp(self.dauer-self.L1_time))
            self.stage=np.random.choice(["L2d","L2"],p=[prob,1-prob])
            print("Worm",self.name,"has reached",self.stage)
        if self.age==40 and self.stage=="L2":
            self.stage="L3"
            print("Worm",self.name,"has reached L3")
        if self.age==45 and self.stage=="L2d":
            prob=1/(1+math.exp(self.dauer-self.L2d_time))
            self.stage=np.random.choice(["dauer","L3"],p=[prob,1-prob])
            print("Worm",self.name,"has reached",self.stage)
            self.age=40
        if self.age==80:
            self.stage="L4"
            print("Worm",self.name,"has reached L4")
        if self.age==160:
            self.stage="adult"
            print("Worm",self.name,"has reached adulthood")
            if self.gender=="male":
                males[self.position].append(self.name)
            if self.gender=="female":
                females[self.position].append(self.name)
        if self.age==320:
            self.stage="old"
            print("Worm",self.name,"is old")
            if self.gender=="male":
                males[self.position].remove(self.name)
            if self.gender=="female":
                females[self.position].remove(self.name)
        if self.age==400:
            print("Worm",self.name,"has died of old age")
            P1.pop[self.name]="dead old worm"
    def reproduce(self):
        if self.stage=="adult" and self.gender=="hermaphrodite":
            count=0
            for i in range(len(P1.pop),len(P1.pop)+10):
                if self.energy>0:
                    self.energy=self.energy-0.5
                    P1.pop.append(Worm(i))
                    P1.pop[i].gender=np.random.choice(["hermaphrodite","male"],p=[0.99,0.01])
                    P1.pop[i].location=[self.location[0],self.location[1]]
                    P1.pop[i].dauer_1=np.random.choice([self.dauer_1,self.dauer_2,np.random.normal(self.dauer_1,0.5),np.random.normal(self.dauer_2,0.5)],p=[0.4995,0.4995,0.0005,0.0005])
                    P1.pop[i].dauer_2=np.random.choice([self.dauer_1,self.dauer_2,np.random.normal(self.dauer_1,0.5),np.random.normal(self.dauer_2,0.5)],p=[0.4995,0.4995,0.0005,0.0005])
                    count=count+1
            print(count,"eggs have been laid at",self.location)
        if self.stage=="adult" and self.gender=="female" and len(self.sperm)>0:
            count=0
            for i in range(len(P1.pop),len(P1.pop)+10):
                if self.energy>0:
                    self.energy=self.energy-0.5
                    P1.pop.append(Worm(i))
                    P1.pop[i].gender=np.random.choice(["male","female"])
                    P1.pop[i].location=[self.location[0],self.location[1]]
                    P1.pop[i].dauer_1=np.random.choice([self.dauer_1,self.dauer_2,np.random.normal(self.dauer_1,0.5),np.random.normal(self.dauer_2,0.5)],p=[0.4995,0.4995,0.0005,0.0005])
                    sperm=[]
                    prob=[]
                    for i in self.sperm:
                        sperm.append(i)
                        sperm.append(np.random.normal(i,0.5))
                        prob.append(0.999/len(self.sperm))
                        prob.append(0.001/len(self.sperm))
                    P1.pop[i].dauer_2=np.random.choice(sperm,p=prob)
                    count=count+1
            print(count,"eggs have been laid at",self.location)
    def decide(self):
        if self.stage=="L1":
            self.L1_time=self.L1_time+1
        if self.stage=="L2d":
            self.L2d_time=self.L2d_time+1
        if self.stage!="egg":
            food_weights=np.round(np.arange(1,-0.0001,-(1/100)),8)
            neighbors_weights=np.arange(0,1.0001,(1/100))
            F=grid[self.position,2]
            F=F.astype(int)
            P=grid[self.position,3]+grid[100*self.location[1]+((self.location[0]+1)%100),3]+grid[100*self.location[1]+((self.location[0]-1)%100),3]+grid[100*((self.location[1]+1)%100)+self.location[0],3]+grid[100*((self.location[1]-1)%100)+self.location[0],3]
            P=np.round(P)
            P=P.astype(int)
            if F==0:
                probability=1
            elif self.energy==0:
                probability=0
            elif P>99:
                probability=(food_weights[F]+1)/2
            else:
                probability=(food_weights[F]+neighbors_weights[P])/2
            if np.random.choice([True,False],p=[probability,1-probability])==True:
                self.move()
            else:
                self.eat()
            self.reproduce()
        if self.stage=="egg":
            print("Worm",self.name,"is an egg")
            self.age=self.age+1
            self.grow()
        if self.stage=="L2d" and (1/(1+math.exp(self.dauer-self.L2d_time)))>0.99:
            self.age=40
            self.stage="dauer"
            print("Worm",self.name,"has reached dauer")

class Population:
    def __init__(self,size):
        self.size=size
        self.pop=[]
    def populate(self):
        for i in range(0,self.size):
            self.pop.append(Worm(i))
            print("The location of",self.pop[i].name,"is",self.pop[i].location)
    def decide(self,worm_lower,worm_upper,decision):
        self.worm_lower=worm_lower
        self.worm_upper=worm_upper
        self.decision=decision
        number=0
        while number<self.decision:
            bounds=np.arange(self.worm_lower,self.worm_upper)
            rnd.shuffle(bounds)
            for i in bounds:
                try:
                    self.pop[i].decide()
                except AttributeError:
                    print("Worm",i,"is dead")
                    continue
            number=number+1
            list_sperm()
        grow_food()
        decay_pheromones()

def worm_map():
    x,y=np.mgrid[slice(0,100),slice(0,100)]
    z=grid[100*y+x,3]
    plt.pcolormesh(x,y,z,cmap='Blues')
    #plt.pcolormesh(x,y,z,cmap='Blues',vmin=0,vmax=100) to normalize
    plt.title('Pheromone Density')
    plt.xlabel('X Coordinate')
    plt.ylabel('Y Coordinate')
    plt.axis([0,100,0,100])
    plt.colorbar()

def food_map():
    x,y=np.mgrid[slice(0,100),slice(0,100)]
    z=grid[100*y+x,2]
    plt.pcolormesh(x,y,z,cmap='Blues')
    #plt.pcolormesh(x,y,z,cmap='Blues',vmin=0,vmax=100) to normalize
    plt.title('Food Density')
    plt.xlabel('X Coordinate')
    plt.ylabel('Y Coordinate')
    plt.axis([0,100,0,100])
    plt.colorbar()

def prob_move():
    x,y=np.mgrid[slice(0,101),slice(0,101)]
    z=np.zeros((101,101))
    for i in range(0,101):
        for j in range(0,101):
            food_weights=np.round(np.arange(1,-0.0001,-(1/100)),8)
            neighbors_weights=np.arange(0,1.0001,(1/100))
            if i==0:
                z[i,j]=1
            elif j>99:
                z[i,j]=(food_weights[i]+1)/2
            else:
                z[i,j]=(food_weights[i]+neighbors_weights[j])/2
    plt.pcolormesh(x,y,z,cmap='Blues')
    plt.title('Probability of Moving')
    plt.xlabel('Amount of Food')
    plt.ylabel('Amount of Pheromones')
    plt.axis([0,100,0,100])
    plt.colorbar(ticks=[np.arange(0,1.1,0.1)])

def how_many():
    egg,L1,L2,L2d,L3,dauer,L4,adult,old,dsw,dow=0,0,0,0,0,0,0,0,0,0,0
    for i in range(len(P1.pop)):
        if P1.pop[i]=="dead starving worm":
            dsw=dsw+1
        if P1.pop[i]=="dead old worm":
            dow=dow+1
        try:
            if P1.pop[i].stage=="egg":
                egg=egg+1
            if P1.pop[i].stage=="L1":
                L1=L1+1
            if P1.pop[i].stage=="L2":
                L2=L2+1
            if P1.pop[i].stage=="L2d":
                L2d=L2d+1
            if P1.pop[i].stage=="L3":
                L3=L3+1
            if P1.pop[i].stage=="dauer":
                dauer=dauer+1
            if P1.pop[i].stage=="L4":
                L4=L4+1
            if P1.pop[i].stage=="adult":
                adult=adult+1
            if P1.pop[i].stage=="old":
                old=old+1
        except AttributeError:
            continue
    print(egg,"egg")
    print(L1,"L1")
    print(L2,"L2")
    print(L2d,"L2d")
    print(L3,"L3")
    print(dauer,"dauer")
    print(L4,"L4")
    print(adult,"adult")
    print(old,"old")
    print(dsw,"dead starving worm")
    print(dow,"dead old worm")

def stats():
    gene_1=[]
    gene_2=[]
    for i in range(len(P1.pop)):
        try:
            gene_1.append(P1.pop[i].dauer_1)
            gene_2.append(P1.pop[i].dauer_2)
        except AttributeError:
            continue
    print("Gene 1 mean",np.mean(gene_1))
    print("Gene 2 mean",np.mean(gene_2))
    print("Gene 1 std",np.std(gene_1))
    print("Gene 2 std",np.std(gene_2))

P1=Population(1000)
P1.populate()


# -*- coding: utf-8 -*-

from zhihu_scrapy import db

zhihu = db.ZhihuMongo("zhihu")

def main():
    while zhihu["waitting_urls"].find_one():
        from scrapy.cmdline import execute
        execute("scrapy crawl zhihu_crawl".split())

if __name__ == "__main__":
    main()


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from random_user_agent.user_agent import UserAgent

from random_user_agent.params import SoftwareName, OperatingSystem
import openpyxl
import random
import time

search_char = ['L','S','T']

software_names = [SoftwareName.CHROME.value]
operating_systems = [OperatingSystem.WINDOWS.value,
                     OperatingSystem.LINUX.value]
user_agent_rotator = UserAgent(software_names = software_names,
                       operating_systems = operating_systems,
                       limit=100)
user_agent = user_agent_rotator.get_random_user_agent()

def click_element(elem_type=0, elem_selector=0):
    if(elem_type == 'css'):
        elem = browser.find_element_by_css_selector(elem_selector)
    elem.click

def sleep_fuc():
    time.sleep(random.randrange(2,4))

def get_search_window(srch_address,srch_prim_plan,srch_sub_plan,row_num,col_num,active_sheet):
    global browser
    global software_names, operating_systems, user_agent_rotator, user_agent

    chrome_options = Options()
    chrome_options.add_argument("user-agent ={}".format(user_agent))

    browser = webdriver.Chrome(options=chrome_options)
    browser.get('https://www.blueshieldca.com/fap/app/find-a-doctor.html')

    browser.switch_to.window(browser.window_handles[1])

    doctor_button=browser.find_element_by_id('Doctor')

    doctor_button.click()

    #guest_button=find_element_by_id('nonMember')
    WebDriverWait(browser,2).until(EC.element_to_be_clickable((By.ID,'nonMember'))).click()

    #guest_button.click()

    location_input=browser.find_element_by_id('autocomplete')

    #location_input.click()
    #location_input.send_keys('Los Angeles, CA 90002, USA')

    sleep_fuc()

    while(browser.current_url != 'https://www.blueshieldca.com/fad/plans/chooseplan'):
        location_input.clear()
        location_input.send_keys(srch_address)

        location_input.send_keys(Keys.LEFT)
        location_input.send_keys(Keys.RIGHT)

        action = webdriver.common.action_chains.ActionChains(browser)
        action.move_to_element_with_offset(location_input, 100,60)
        action.click()
        action.perform()

        sleep_fuc()

        #CLICK CONTINUE
        #click_element('css','#main > div > app-location > div.location-container > div > div.wrapper-location.col-sm-12.col-md-10.col-lg-7 > button')
        continue_button = browser.find_element_by_css_selector('#main > div > app-location > div.location-container > div > div.wrapper-location.col-sm-12.col-md-10.col-lg-7 > button')
        sleep_fuc()

        continue_button.click()

    sleep_fuc()
    for tries in range(0,5):
        try:
            browser.refresh()
            allHTML = browser.find_element_by_tag_name('html')
            allHTML.send_keys(Keys.PAGE_DOWN)
            time.sleep(1)
            primary_plan_button = browser.find_element_by_css_selector('#main > div > app-choose-plans > div > div > div > div > div > button.btn.btn-primary.planButtonLeft')
            primary_plan_button.click()

            #CLICK PRIMARY PLAN DROPDOWN
            #click_element('css','#primaryPlanDropdown > div')
            #htmlElem = browser.find_element_by_tag_name('html')
            #htmlElem.send_keys(Keys.PAGE_DOWN)
            primary_plan_dropdown = browser.find_element_by_xpath('//*[@id="primaryPlanDropdown"]')
            browser.execute_script("arguments[0].click();", primary_plan_dropdown)
            time.sleep(2)

            primary_plan_ul = browser.find_element_by_xpath('//*[@id="main"]/div/app-plans/div[1]/div/div/div/form/div/div/div/div[3]/div[2]/ul')
            primary_plans = primary_plan_ul.find_elements_by_tag_name('li')

            srch_prim_plan = srch_prim_plan.replace(" ","").lower()
            for plan in primary_plans:
                plan_text = plan.text
                plan_text = plan_text.replace(" ","").lower()
                if (plan_text == srch_prim_plan):
                    plan.click()
                    break

            try:
                sub_plan_dropdown = browser.find_element_by_xpath('//*[@id="subPlanDropdown"]')
                browser.execute_script("arguments[0].click();", sub_plan_dropdown)

                allHTML = browser.find_element_by_tag_name('html')
                allHTML.send_keys(Keys.PAGE_DOWN)
                time.sleep(1)

                sub_plan_ul = browser.find_element_by_xpath('//*[@id="main"]/div/app-plans/div[1]/div/div/div/form/div/div/div/div[3]/div[3]/ul')
                sub_plans = sub_plan_ul.find_elements_by_tag_name('li')

                #Create search pattern

                srch_sub_plan = srch_sub_plan.replace(" ","")

                for sub in sub_plans:
                    sub_text = sub.text
                    sub_text = sub_text.replace(" ","")

                    if (sub_text == srch_sub_plan):
                        sub.click()
                        break

            except:
                print('No Subplan')

            sleep_fuc()
            plan_continue = browser.find_element_by_xpath('//*[@id="continuePlan"]')
            browser.execute_script("arguments[0].click();", plan_continue)
            sleep_fuc()
            return True
        except Exception as e:
            #print(e)
            print('Request Denied, try# '+str(tries))
            #browser.quit()
            #software_names = [SoftwareName.CHROME.value]
            #operating_systems = [OperatingSystem.WINDOWS.value,
                                 #OperatingSystem.LINUX.value]
            #user_agent_rotator = UserAgent(software_names = software_names,
                                   #operating_systems = operating_systems,
                                   #limit=100)
            #user_agent = user_agent_rotator.get_random_user_agent()
            browser.back()
            time.sleep(2*tries)
            #get_search_window(srch_address,srch_prim_plan,srch_sub_plan,row_num,col_num,active_sheet)
            #active_sheet.cell(row = row_num, column = col_num).value = "0 (Manually Check)"
    return False
#####################

def get_doctor_count(row_num,col_num,active_sheet):
    global search_char
    #doctor_name = browser.find_element_by_xpath('//*[@id="Enter last name"]/label')
    #doctor_name.click()

    WebDriverWait(browser,2).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="Enter last name"]/label'))).click()
    time.sleep(0.3)

    enter_name = WebDriverWait(browser,2).until(EC.presence_of_element_located((By.XPATH,'//*[@id="provider_name"]')))
    enter_name.clear()
    enter_name.send_keys(search_char[0])

    search_btn = browser.find_element_by_xpath('//*[@id="searchBtn"]')
    search_btn.click()

    sleep_fuc()

    try:
        miles = browser.find_element_by_xpath('//*[@id="searchResultMsg"]/a')
        browser.execute_script("arguments[0].click();", miles)
    except:
        total_doctors = "0 (Failed at miles)"
        active_sheet.cell(row = row_num, column = col_num).value = total_doctors
        print('Failed at miles')
        return

    miles_dropdown = browser.find_element_by_xpath('//*[@id="dropdownRadiusFilter"]')
    browser.execute_script("arguments[0].click();", miles_dropdown)

    time.sleep(1.5)

    #miles_ul = browser.find_element_by_xpath('//*[@id="affix_el"]/app-filtersort/div/div[2]/form/fieldset/div[1]/div[1]/div/ul')
    miles_100 = browser.find_element_by_xpath('//*[@id="select_lo7"]')
    miles_100.click()

    apply_btn = browser.find_element_by_xpath('//*[@id="filter-button-apply"]')
    apply_btn.click()
    time.sleep(4)

    try:
        total_doctors = browser.find_element_by_xpath('//*[@id="searchResultCount"]/span[1]')
    except:
        total_doctors = "0"

    active_sheet.cell(row = row_num, column = col_num).value = total_doctors.text
    print('Success!: '+ str(row_num)+', '+str(col_num))

    for index in range(1,3):
        col_num = col_num + 1
        enter_name = WebDriverWait(browser,2).until(EC.presence_of_element_located((By.XPATH,'//*[@id="provider_name"]')))
        enter_name.clear()
        enter_name.send_keys(search_char[index])

        WebDriverWait(browser,2).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="searchBtn"]'))).click()
        time.sleep(5)
        try:
            total_doctors = browser.find_element_by_xpath('//*[@id="searchResultCount"]/span[1]')
        except:
            total_doctors = "0"

        active_sheet.cell(row = row_num, column = col_num).value = total_doctors.text
        print('Success!: '+ str(row_num)+', '+str(col_num))

####################

def main():
    #OPEN EXCEL FILE
    #print('Please input the whole file name. (must be .xlsx file)')
    file = 'Site 5 Product Research Jun21.xlsx'
    wb = openpyxl.load_workbook(file)

    print('Start on which row?')
    start_row = input()
    print('Start on which column?')
    start_column = input()

    sheet = wb['Sheet1']

    for row in range(start_row, sheet.max_row +1):
        sub_plan = sheet['D' + str(row)].value
        prim_plan = sheet['E' + str(row)].value

        for col in range(start_column, sheet.max_column + 1, 3):
            search_value = sheet.cell(row = 1, column = col).value
            zip_code = search_value[0:5]
            #search_char = search_value[-1]

            sheet = wb['Sheet2']
            for zip_row in range(1,sheet.max_row+1):
                if (str(zip_code) == str(sheet['A'+str(zip_row)].value)):
                    address = sheet['B'+str(zip_row)].value
                    break

            sheet = wb['Sheet1']
            proceed = get_search_window(address,prim_plan,sub_plan,row,col,sheet)
            if (proceed):
                get_doctor_count(row,col,sheet)
            wb.save(file)
            browser.quit()

        start_column = 7

    print('Done!')

main()


# Information: https://clover.coex.tech/en/snippets.html#navigate_wait

import math
import rospy
from clover import srv
from std_srvs.srv import Trigger
import tf2_geometry_msgs
from tf.transformations import euler_from_quaternion

#rospy.init_node('flight')

get_telemetry = rospy.ServiceProxy('get_telemetry', srv.GetTelemetry)
navigate = rospy.ServiceProxy('navigate', srv.Navigate)
navigate_global = rospy.ServiceProxy('navigate_global', srv.NavigateGlobal)
set_position = rospy.ServiceProxy('set_position', srv.SetPosition)
set_velocity = rospy.ServiceProxy('set_velocity', srv.SetVelocity)
set_attitude = rospy.ServiceProxy('set_attitude', srv.SetAttitude)
set_rates = rospy.ServiceProxy('set_rates', srv.SetRates)
land = rospy.ServiceProxy('land', Trigger)

def navigate_wait(x=0, y=0, z=0, yaw=float('nan'), yaw_rate=0, speed=1, \
        frame_id='body', tolerance=0.2, auto_arm=False):

    res = navigate(x=x, y=y, z=z, yaw=yaw, yaw_rate=yaw_rate, speed=speed, \
        frame_id=frame_id, auto_arm=auto_arm)

    if not res.success:
        return res

print ("Take off 1 meter")
navigate_wait(z=2, frame_id='body', auto_arm=True)
# Wait for 3 seconds
print("Done")
rospy.sleep(3)

# Fly forward 1 m
#navigate_wait(x=1, frame_id='body')

# Land
#land()

def goal_republish(posedata):

    print("Received Simple")
    #do the transformation
    euler = euler_from_quaternion([posedata.pose.orientation.x, posedata.pose.orientation.y, posedata.pose.orientation.z, posedata.pose.orientation.w])
    print("Received Simple Goal: x=",posedata.pose.position.x," y=",posedata.pose.position.y," z=",posedata.pose.position.z," yaw =",euler[2]," frameID=",posedata.header.frame_id)
    navigate_wait( x=posedata.pose.position.x, y=posedata.pose.position.y, z=2, yaw=euler[2], frame_id=posedata.header.frame_id)
    print("Done")
    #rospy.sleep(3)

if __name__ == '__main__':

    rospy.init_node('republisher', anonymous=False)
    rospy.Subscriber("/move_base_simple/goal", tf2_geometry_msgs.PoseStamped, goal_republish)
    rospy.spin()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 10 10:49:28 2021

@author: t1
"""

import time
import numpy as np
from collections import Counter

def time_it(func):
    def wrapper(*args,**kwargs):
        start = time.clock()
        result = func(*args,**kwargs)
        end = time.clock()
        print(' {} executed in time : {:.6f} sec'.format(func.__name__,end - start))
        return result
    return wrapper

def read_file(file_path):
    M = None
    N = None
    spectrum = None
    with open(file_path) as f:
        lines = f.readlines()
        M = int(lines[0].strip())
        N = int(lines[1].strip())
        spectrum = [int(x) for x in lines[2].strip().split(' ')]
    return M,N,sorted(spectrum)

def get_convolution2(spectrum):
    '''
    spectrum -> list of ints

    returns :
        conv_list -> list of ints; convolution of spectrum
    '''
    spec_arr = np.array(spectrum)
    conv_mat = spec_arr.reshape(-1,1) - spec_arr.reshape(1,-1)
    conv_list = list(conv_mat[conv_mat > 0].ravel())
    return conv_list

def get_top_M_weights(M,conv_list):
    '''
    M -> int
    conv_list -> list of ints ;

    returns -> top M most common values in convolution
    '''
    conv_list = [el for el in conv_list if (el >= 57) and (el <= 200)]
    counts_ = Counter(conv_list)
    mc_list = counts_.most_common(M)
    last_count_val = mc_list[-1][1]
    for (key,val) in counts_.items():
        if (val == last_count_val) and ((key,val) not in mc_list):
            mc_list.append((key,val))
    return [w for (w,c) in mc_list]

def expand_cands(cand_list,mass_list):
    '''
    cand_list -> list of strs

    returns new collection containing all possible extensions of
    peptides in Peptides by a single amino acid mass
    '''
    new_cand_list = []
    for cand in cand_list:
        for mass in mass_list:
            if cand == "":
                new_cand = str(mass)
            else:
                new_cand = cand + '-' + str(mass)
            new_cand_list.append(new_cand)
    return new_cand_list

def get_cand_mass(cand):
    '''
    cand -> str
    returns total mass for candidate sequence
    '''
    mass_list = [int(m) for m in cand.split('-')]
    return sum(mass_list)

def get_cand_spectrum(cand):

    '''
    cand -> str of masses eg : '113-129-128-114'
    returns cyclo spectrum of peptides
    '''
    mass_list = [int(m) for m in cand.split('-')]
    spectrum = []
    for mass in mass_list:
        spectrum.append(mass)
    for k in range(2,len(mass_list)):
        for i in range(len(mass_list)):
            j = i+k
            w_ = None
            if j > len(mass_list):
                w_ = sum(mass_list[i:]) + sum(mass_list[:(j - len(mass_list))])
            else:
                w_ = sum(mass_list[i:j])
            spectrum.append(w_)
    if len(mass_list) > 1:
        spectrum.append(sum(mass_list))
    spectrum = [0] + spectrum
    return sorted(spectrum)

def get_cand_lin_spectrum(cand):
    mass_list = [int(m) for m in cand.split('-')]
    spectrum = []
    for mass in mass_list:
        spectrum.append(mass)
    for k in range(2,len(mass_list)):
        for i in range(len(mass_list) -k +1):
            j = i+k
            w_ = sum(mass_list[i:j])
            spectrum.append(w_)
    if len(mass_list) > 1:
        spectrum.append(sum(mass_list))
    spectrum = [0] + spectrum
    return sorted(spectrum)

def get_cand_lin_score(peptide,spectrum):
    '''
    peptide -> str
    spectrum -> list of ints
    score is defined as the number of masses shared between
    Cyclospectrum(Peptide) and Spectrum
    '''
    peptide_spectrum = get_cand_lin_spectrum(peptide)
    pep_spectrum_set = list(set(peptide_spectrum))
    score = 0
    for mass in pep_spectrum_set:
        count_1 = peptide_spectrum.count(mass)
        count_2 = spectrum.count(mass)
        score += min(count_1,count_2)
    return score

def get_cand_cyclic_score(peptide,spectrum):
    '''
    peptide -> str
    spectrum -> list of ints
    score is defined as the number of masses shared between
    Cyclospectrum(Peptide) and Spectrum
    '''
    peptide_spectrum = get_cand_spectrum(peptide)
    pep_spectrum_set = list(set(peptide_spectrum))
    score = 0
    for mass in pep_spectrum_set:
        count_1 = peptide_spectrum.count(mass)
        count_2 = spectrum.count(mass)
        score += min(count_1,count_2)
    return score

def trim(N,lead_board,spectrum):
    if len(lead_board) == 0:
        return []
    score_list = [(cand,get_cand_lin_score(cand, spectrum)) for cand in lead_board]
    sorted_score_list = sorted(score_list,key = lambda x : x[1],reverse = True)
    # print(sorted_score_list)
    trimmed_list = sorted_score_list[:N]
    nt_score = trimmed_list[-1][1]
    for cand,score in sorted_score_list[N:]:
        if score == nt_score:
            trimmed_list.append((cand,score))
    trimmed_cand_list = [x[0] for x in trimmed_list]
    # print(len(trimmed_cand_list))
    return trimmed_cand_list

def get_cand_peptide(N,spectrum,mass_list):
    '''
    N -> leaderboard size
    spectrum -> experimental spectrum
    returns peptide cancdidate with the highest score
    '''

    lead_board = [""]
    lead_board_trim = []
    lead_cand = ''
    lead_score = 0
    cand_score_dict = {}
    PARENT_MASS = max(spectrum)
    while len(lead_board) > 0:
        lead_board = expand_cands(lead_board,mass_list)
        lead_board_trim = []

        for cand in lead_board:
            cand_mass = get_cand_mass(cand)
            # cand_score = get_cand_cyclic_score(cand, spectrum)
            # cand_score_dict[cand] = cand_score
            if cand_mass == PARENT_MASS:
                cand_score = get_cand_cyclic_score(cand, spectrum)
                cand_score_dict[cand] = cand_score

                if cand_score > lead_score:
                    lead_cand = cand
                    lead_score = cand_score
            if cand_mass <= PARENT_MASS:
                lead_board_trim.append(cand)
        lead_board = trim(N,lead_board_trim,spectrum)

    return lead_cand

@time_it
def get_conv_cyclo_pep_seq(M,N,spectrum):
    '''
    M -> int ; top M most frequent weight values are taken from convolution of spectrum
    N -> int ; trim top N values in leaderboard cyclopeptide sequencing
    spectrum -> list of ints ; experimental spectrum

    returns : most probable sequence of peptide
    '''
    # get spectrum convolution
    conv_list = get_convolution2(spectrum)
    mass_list = get_top_M_weights(M, conv_list)
    # print('masses : ',len(mass_list),mass_list)

    lead_cand = get_cand_peptide(N, spectrum, mass_list)
    return lead_cand

# file_path = 'sample_data/convol_cyclo_pep_seq.txt'
file_path = 'test_data/dataset_104_7.txt'
M,N,spectrum = read_file(file_path)
lead_cand = get_conv_cyclo_pep_seq(M, N, spectrum)
print(lead_cand)


import github_repo as gh
import json

OWNER = 'glass-bead-labs'
REPO = 'sensor-group'

sensor_group = gh.GitHubRepo(OWNER, REPO)

data = {}
dataset = []

for label in sensor_group.get_all_labels():
    issues = []
    for issue in sensor_group.get_issues_with_label(label):
        index = sensor_group.get_index(issue)
        size = sensor_group.get_num_comments(index) + 1
        issues.append({"name": issue, "size": size})
    dataset.append({"name": label, "children": issues})

issues = []
for issue in sensor_group.get_issues_without_label():
    index = sensor_group.get_index(issue)
    size = sensor_group.get_num_comments(index) + 1
    issues.append({"name": issue, "size": size})

dataset.append({"name": "no label", "children": issues})
data["name"] = OWNER + '/' + REPO
data["children"] = dataset

# Creates another JSON file.
with open ('bubble.json', 'w') as outfile:
    json.dump(data, outfile, indent=4)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 10 10:30:09 2019

@author: koch
"""

# Import libraries

# Misc
from pathlib import Path
from os import listdir
from os import system
from copy import deepcopy
from sys import argv

# Math
import numpy as np
import scipy.io
import scipy.stats
from random import sample
from collections import Counter

# Image processing
import nibabel
from nilearn import signal
from nilearn import image
import pandas as pd

# Machine learning
import sklearn.model_selection
import sklearn.svm

# Plotting
import matplotlib.pyplot as plt
from matplotlib import gridspec

''' # Define functions used in this script '''
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# 1. Create_raw_matrix
# Function to create matrix for classifier test on raw data (each TR one row)
# - Define path to raw data
# - Load smoothed or unsmoothed raw_data
# - Transform into test matrix (2D array: 1st D voxel array, 2nd D TR)
# - Return test matrix of raw data for each run and mask in list of lists
#       - 1st list: Run 1, Run 2
#       - 2nd lst: Mask 1, ..., Mask n
#       - Example: output[0][4] = raw_mat of Run 1, 5th Mask
#       - Example: output[1][0] = raw_mat of Run 2, 1st Mask

# Parameters:
# 1. bids_dir: Path to BIDS structured directory
# 2. sub_id: participant code
# 3. masks: 3D array of mask values in 1s and 0s, 4D array in case of
# multiple masks
# 4. preproc: Logical to pick preprocessed or unprocessedraw data
#       - preproc already corrected, smoothed, and z-scored the data and
#       eliminated TRs which are not in the behavioral data
#       - 0: Use unprocessed data
#       - 1: Use preprocessed data
# 5. smooth: In case of unprocessed data. Boolean to determine if data should
# be smoothed
#       - 0: Data will not be smoothed
#       - 1: Data will be smoothed
# 6. smooth_kernel: Number to determine kernel size of smoothing (in case
# data will be smoothed)
# 7. concat: Boolean to determine if separated runs should be returned
# concatenated or as separated runs (in list format)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def Create_raw_matrix(bids_dir, sub_id, masks, preproc, smooth, smooth_kernel,
                      concat):

    # Pose input of masks in appropriate format in case only one mask is
    # given
    if len(masks.shape) == 3:
        # Add 4th dimension and fill first 3 with provided mask
        pattern = np.zeros([masks.shape[0], masks.shape[1], masks.shape[2], 1])
        pattern[:,:,:,0] = masks
        # Make new 4D array to masks
        masks = pattern

    # Provide path to raw data dependent on taking preprocessed or unprocessed
    # data
    if bool(preproc):
        raw_dir_r1 = (bids_dir + '/derivatives/' + sub_id +
                      '/preproc/classification_raw/zscsar' + sub_id +
                      '_task-feedback1_bold.nii')
        raw_dir_r2 = (bids_dir + '/derivatives/' + sub_id +
                      '/preproc/classification_raw/zscsar' + sub_id +
                      '_task-feedback2_bold.nii')
    else:
        raw_dir_r1 = (bids_dir + '/derivatives/' + sub_id +
                      '/preproc/smooth/sar' + sub_id +
                      '_task-feedback1_bold.nii')
        raw_dir_r2 = (bids_dir + '/derivatives/' + sub_id +
                      '/preproc/smooth/sar' + sub_id +
                      '_task-feedback2_bold.nii')

    # Load selected raw data into matrix
    nii_raw_r1 = nibabel.load(raw_dir_r1)
    nii_raw_r2 = nibabel.load(raw_dir_r2)

    # Smooth data with provided kernel size in case unprocessed data is loaded
    if bool(smooth):
        nii_raw_r1 = image.smooth_img(nii_raw_r1, smooth_kernel)
        nii_raw_r2 = image.smooth_img(nii_raw_r2, smooth_kernel)

    # Cast nii data into matrix
    volume_raw_r1 = nii_raw_r1.get_data()
    volume_raw_r2 = nii_raw_r2.get_data()

    # Ger number of TRs in each run
    n_tr_r1 = volume_raw_r1.shape[3]
    n_tr_r2 = volume_raw_r2.shape[3]

    # Allocate list to store raw matrices for different masks
    masked_raw_mat_r1 = [0] * masks.shape[3]
    masked_raw_mat_r2 = deepcopy(masked_raw_mat_r1)
    # Allocate list for concatenated runs
    masked_raw_mat_concat = deepcopy(masked_raw_mat_r1)

    # Create different raw matrix for each mask
    for mask_count in np.arange(masks.shape[3]):

        # Get current mask
        current_mask = masks[:,:,:,mask_count]

        # Allocate 2D array: 1st D holding all non-zero masked voxels, 2nd D
        # holding different TRs
        current_raw_mat_r1 = np.zeros([np.flatnonzero(current_mask).size,
                                       n_tr_r1])
        current_raw_mat_r2 = np.zeros([np.flatnonzero(current_mask).size,
                                       n_tr_r2])

        # Fill allocated raw matrices with masked TRs (Separated for runs
        # since number of TRs differs)
        # Run 1
        for tr_count in np.arange(volume_raw_r1.shape[3]):

            # Isolate current TR
            current_tr = volume_raw_r1[:,:,:,tr_count]

            # Fill raw matrix with masked values for current TR
            current_raw_mat_r1[:,tr_count] = (
                    current_tr[np.nonzero(current_mask)]
                    )

        # Run 2
        for tr_count in np.arange(volume_raw_r2.shape[3]):
            current_tr = volume_raw_r2[:,:,:,tr_count]
            current_raw_mat_r2[:,tr_count] = (
                    current_tr[np.nonzero(current_mask)]
                    )

        # Transpose to make different TRs the rows of the matrix
        current_raw_mat_r1 = np.transpose(current_raw_mat_r1, (1,0))
        current_raw_mat_r2 = np.transpose(current_raw_mat_r2, (1,0))

        # Fill list of raw matrices with raw matrix for current mask
        masked_raw_mat_r1[mask_count] = current_raw_mat_r1
        masked_raw_mat_r2[mask_count] = current_raw_mat_r2

        # Create concatenated runs to return
        masked_raw_mat_concat[mask_count] = np.concatenate(
                [masked_raw_mat_r1[mask_count],
                masked_raw_mat_r2[mask_count]])

    # Check for concatenate variable
    # Do not concatenate runs
    if bool(not concat):
        # Return complete list of masked raw matrices in list form
        return(masked_raw_mat_r1, masked_raw_mat_r2)
    elif bool(concat):
        # Return concatenated runs
        return(masked_raw_mat_concat)

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# 2. Create_beta_matrix
# Function to create input matrix for classifier training on betas
# - Define path to betas
# - Load preprocessed betas
# - Transform into input matrix (1D voxel array)
# - Return input matrix for both buffers in list of lists
#       - 1st list: buffer 1, buffer 2
#       - 2nd lst: Mask 1, ..., Mask n
#       - Example: output[0][4] = beta_mat of Buffer1, 5th Mask
#       - Example: output[1][0] = beta_mat of Buffer2, 1st Mask

# Parameters:
# 1. bids_dir: Path to BIDS structured directory
# 2. sub_id: participant code
# 3. masks: 3D array of mask values in 1s and 0s, 4D array in case of
# multiple masks
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def Create_beta_matrix(bids_dir, sub_id, masks):

    # Pose input of masks in appropriate format in case only one mask is
    # given
    if len(masks.shape) == 3:
        # Add 4th dimension and fill first 3 with provided mask
        pattern = np.zeros([masks.shape[0], masks.shape[1], masks.shape[2], 1])
        pattern[:,:,:,0] = masks
        # Make new 4D array to masks
        masks = pattern

    # Define path which contains all corrected, smoothed, and z-scored  betas:
    corr_dir_b1 = (bids_dir + '/derivatives/' + sub_id +
                   '/glm/buffer1' + '/estimate/corr')
    corr_dir_b2 = (bids_dir + '/derivatives/' + sub_id +
                   '/glm/buffer2' + '/estimate/corr')

    # Give number of betas to load (skipping motion and mean related betas)
    req_betas = np.concatenate([np.arange(1,13),
                                np.arange(19,31)])

    # Allocate 4D array to store betas, dim 1-3: Voxel values, dim 4:
    # different betas
    beta_mat_b1 = np.zeros([masks.shape[0],
                            masks.shape[1],
                            masks.shape[2],
                            len(req_betas)])
    beta_mat_b2 = deepcopy(beta_mat_b1)

    # Load unmasked betas from both buffers
    for beta_count, beta_id in enumerate(req_betas):

        # Path to load specific beta
        path_b1 = corr_dir_b1 + '/zscbeta_' + str(beta_id).zfill(4) + '.nii'
        path_b2 = corr_dir_b2 + '/zscbeta_' + str(beta_id).zfill(4) + '.nii'

        # Load beta .nii with nibabel function
        nii_beta_b1 = nibabel.load(path_b1)
        nii_beta_b2 = nibabel.load(path_b2)

        # Transform nii file into matrix:
        volume_b1 = nii_beta_b1.get_data()
        volume_b2 = nii_beta_b2.get_data()

        # Concatenate beta into 4D array
        beta_mat_b1[:,:,:,beta_count] = volume_b1
        beta_mat_b2[:,:,:,beta_count] = volume_b2

    # Mask betas and pose into list with one entry for each mask
    # Create list of beta matrices with length of number of provided masks
    masked_beta_mat_b1 = [0] * masks.shape[3]
    masked_beta_mat_b2 = deepcopy(masked_beta_mat_b1)

    for mask_count in np.arange(masks.shape[3]):

        # Get current mask
        current_mask = masks[:,:,:,mask_count]

        # Allocate 2D array: 1st D holding all non-zero masked voxels, 2nd D
        # holding different betas
        current_beta_mat_b1 = np.zeros([np.flatnonzero(current_mask).size,
                                        len(req_betas)])
        current_beta_mat_b2 = deepcopy(current_beta_mat_b1)

        # Fill current beta mat with masked voxels for each beta
        for beta_count, beta_id in enumerate(req_betas):

            # Use mask to eliminate all values outside of the mask and
            # bring values into 1-D array for buffer 1 and 2
            masked_beta_b1 = beta_mat_b1[:,:,:,beta_count]
            masked_beta_b1 = masked_beta_b1[np.nonzero(current_mask)]

            masked_beta_b2 = beta_mat_b2[:,:,:,beta_count]
            masked_beta_b2 = masked_beta_b2[np.nonzero(current_mask)]

            # Fill current beta_mat
            current_beta_mat_b1[:,beta_count] = masked_beta_b1
            current_beta_mat_b2[:,beta_count] = masked_beta_b2

        # Transpose to make different betas the rows of the matrix
        current_beta_mat_b1 = np.transpose(current_beta_mat_b1, (1,0))
        current_beta_mat_b2 = np.transpose(current_beta_mat_b2, (1,0))

        # Fill list of different finished beta_mats for each mask
        masked_beta_mat_b1[mask_count] = current_beta_mat_b1
        masked_beta_mat_b2[mask_count] = current_beta_mat_b2

    # Return complete list of masked beta matrices in list form
    return(masked_beta_mat_b1, masked_beta_mat_b2)

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# Function to create 4D matrix of masks
# - Define path to mask directory
# - Load masks as nii files
# - Transform into 3D arrays
# - Append 3D arrays of masks to 4D array
# - Return 4D matrix of masks

# Parameters:
# 1. bids_dir: Path to BIDS structured directory
# 2. sub_id: participant code
# 3. mask_names: Array of strings with mask names
# 4. buffering: Logical indicating if to return one mask for each buffer or
# across buffer masks (shared mask)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def Load_masks(bids_dir, sub_id, mask_names, buffering):

    # Allocate list to safe mask matrixes in
    all_masks_b1 = [0] * len(mask_names)
    all_masks_b2 = deepcopy(all_masks_b1)

    # Loop over each mask
    for mask_count, mask_id in enumerate(mask_names):

        # Get path to mask
        mask_dir = bids_dir + '/derivatives/' + sub_id + '/preproc/seg'

        # Create mask path for both buffers
        path_mask_b1 = (mask_dir + '/' + sub_id + '_mask_' + mask_id +
                        '_buffer1.nii')
        path_mask_b2 = (mask_dir + '/' + sub_id + '_mask_' + mask_id +
                        '_buffer2.nii')

        # Load mask to .nii object
        mask_b1 = nibabel.load(path_mask_b1)
        mask_b2 = nibabel.load(path_mask_b2)

        # Extract matrix from .nii
        mask_b1 = mask_b1.get_data()
        mask_b2 = mask_b2.get_data()

        # Post matrices into lists
        all_masks_b1[mask_count] = mask_b1
        all_masks_b2[mask_count] = mask_b2

    # Post list into array format with 4th D being different masks
    # Create empty 4D array with correct dimensions (given by first mask)
    all_masks_mat_b1 = np.zeros([all_masks_b1[0].shape[0],
                                 all_masks_b1[0].shape[1],
                                 all_masks_b1[0].shape[2],
                                 len(mask_names)])
    all_masks_mat_b2 = deepcopy(all_masks_mat_b1)

    # Fill 4D array with masks from lists
    for mask_count, mask_id in enumerate(mask_names):
        all_masks_mat_b1[:,:,:,mask_count] = all_masks_b1[mask_count]
        all_masks_mat_b2[:,:,:,mask_count] = all_masks_b2[mask_count]

    # Return buffer specific masks if indicated by 'buffering' parameter
    if bool(buffering):
        return(all_masks_mat_b1, all_masks_mat_b2)

    # In case combined mask is required create combined mask
    else:
        # Allocate empty 4d array holding combined masks
        all_masks_mat = np.zeros([all_masks_b1[0].shape[0],
                                 all_masks_b1[0].shape[1],
                                 all_masks_b1[0].shape[2],
                                 len(mask_names)])

        # Fill empty 4D array with combined masks
        for mask_count, mask_id in enumerate(mask_names):
            # Allocate empty mask array same shape as other masks
            mask = np.zeros([all_masks_b1[0].shape[0],
                                 all_masks_b1[0].shape[1],
                                 all_masks_b1[0].shape[2]])

            # Fill empty mask with 1 at every coordinate where both bufferd
            # masks are 1 (shared mask)
            shared_voxels = np.where(all_masks_mat_b1[:,:,:,mask_count] +
                                     all_masks_mat_b2[:,:,:,mask_count] == 2)
            mask[shared_voxels] = 1

            # Fill empty array holding all shared masks with current mask
            all_masks_mat[:,:,:,mask_count] = mask

        # Return shared masks in 4D array
        return(all_masks_mat)

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# Function to load behavioral data about runs
# - Load behavioral data about runs
# - Option to relate run1 to run2 (e.g. if last TR run 1 = 100, First TR of
# run 2 = 101)
# - Option to concatenate behavioral data
# - Add overall event counter

# Parameters:
# 1. bids_dir: Path to BIDS structured directory
# 2. sub_id: participant code
# 3. relate: Boolean if both runs should be ralted to each other (e.g. )
# 4. concat: Boolean to determine if separated runs should be returned
# concatenated or as separated runs (in list format)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def Load_behavioral(bids_dir, sub_id, relate, concat):

    # Paths to both behavioral runs
    path_r1 = (bids_dir + '/derivatives/' + sub_id + '/behavior/' + sub_id +
               '_3d1_backwards.tsv')
    path_r2 = (bids_dir + '/derivatives/' + sub_id + '/behavior/' + sub_id +
               '_3d2_backwards.tsv')

    # Load behavioral .tsv files
    behav_r1 = pd.read_csv(path_r1, sep='\t')
    behav_r2 = pd.read_csv(path_r2, sep='\t')

    # If both runs should be related
    if bool(relate):

        # Make timing relative to previous timing
        behav_r2.loc[:, 'Time'] += behav_r1['Time'].iloc[-1]

        # Make TRs relative to first run
        behav_r2.loc[:, 'TR number corrected'] += (
                behav_r1['TR number uncorrected'].iloc[-1])
        behav_r2.loc[:, 'TR number uncorrected'] += (
                behav_r1['TR number uncorrected'].iloc[-1])

    # If both runs should be concatenated
    if bool(concat):

        # Concatenate both runs
        frames = [behav_r1, behav_r2]
        behav = pd.concat(frames)

        # Return concatenated runs
        return(behav)

    # If runs should be returned separately
    elif bool(not concat):
        return(behav_r1, behav_r2)

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# Function to load motion data of subject
# - Load motion data about runs stemming from realignment
# - Add overall event counter

# Parameters:
# 1. bids_dir: Path to BIDS structured directory
# 2. sub_id: participant code
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def Load_motion(bids_dir, sub_id):

    # Paths to both behavioral runs
    path_r1 = (bids_dir + '/derivatives/' + sub_id + '/preproc/realign/' +
               sub_id + '_mov_reg1.txt')
    path_r2 = (bids_dir + '/derivatives/' + sub_id + '/preproc/realign/' +
               sub_id + '_mov_reg2.txt')

    # Load behavioral .tsv files
    motion_r1 = pd.read_csv(path_r1, delimiter=r"\s+", header=None)
    motion_r2 = pd.read_csv(path_r2, delimiter=r"\s+", header=None)

    return(motion_r1, motion_r2)

''' # Start decoding script '''

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
# Function to decode and predict direction
# - Load all data required (motion, raw, betas, behavioral)
# - Train and test classifier for each mask and save accuracy and predictions
# - Train classifier on permuted labels and save accuracy and predictions

# Parameters:
# 1. bids_dir: BIDS structured directory
# 2. sub_id: Participant code (as string)
# 3. mask_names: List of stings with mask codes
# 4. n_permutation: Number of permutations
# 5. out_dir: Output directory to save results
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #

def main(bids_dir, sub_id, mask_names, n_permutation, out_dir):

    #- - - - - - - - - - - - - - - - -
    # To select certain subjects:
    #sub_list = sub_list[0:1]
    #- - - - - - - - - - - - - - - - -

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # Construct loop to let the classification run for separate masks
    #mask_names = ['beta',
    #              'entorhinal',
    #              'hc',
    #              'm1',
    #              'rsc',
    #              'subiculum',
    #              'thal',
    #              'v1',
    #              'ent_hc']
    # Alternative mask_names for only specific masks:
    #mask_names = ['v1', 'rsc', 'm1']

    # Df to store predictions in (1st C: Subjects, 2nd C: Mask, ...) for real
    # classification and permutation
    clf_prediction = pd.DataFrame()
    clf_prediction_permut = pd.DataFrame()
    # Df to store acuracy in
    clf_accuracy = pd.DataFrame()
    clf_accuracy_permut = pd.DataFrame()

    # Give message to user which subject is investigated
    print('------------------------------------------------------------------')
    print('Working on subject:\t\t\t--|', sub_id, '|--')
    print('------------------------------------------------------------------')

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Load required data
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # Give message to user
    print('\tLoading required data...')

    # Unbuffered (shared mask array)
    mask_mat = Load_masks(bids_dir, sub_id, mask_names, buffering=0)
    # Buffered (two seperate mask arrays)
    mask_mat_b1, mask_mat_b2 = Load_masks(bids_dir, sub_id, mask_names,
                                          buffering=1)

    # Load betas for each buffer into 2D matrix (rows: betas, cols: voxels)
    beta_mat_b1, beta_mat_b2 = Create_beta_matrix(bids_dir, sub_id, mask_mat)

    # Relate betas to directions
    labels_beta = pd.DataFrame()
    labels_beta['angle_bin'] = np.tile(np.arange(1,7), 4)
    labels_beta['fold'] = np.repeat(np.arange(1,5), 6)

    # Load raw data into same format as beta matrix, masked by shared mask
    raw_mat_r1, raw_mat_r2 = Create_raw_matrix(bids_dir, sub_id, mask_mat,
                                               preproc=0,
                                               smooth=1, smooth_kernel=3,
                                               concat=0)
    raw_mat = Create_raw_matrix(bids_dir, sub_id, mask_mat,
                                           preproc=0,
                                           smooth=1, smooth_kernel=3,
                                           concat=1)

    # Load subject motion from realignment for signal.claen
    motion_r1, motion_r2 = Load_motion(bids_dir, sub_id)

    # Signal clean raw mat runs for each mask
    raw_mat_r1 = list(
            signal.clean(x,
                         confounds=motion_r1.values,
                         high_pass = 1/128,
                         t_r = 2.4)
            for x in raw_mat_r1
            )
    raw_mat_r2 = list(
            signal.clean(x,
                         confounds=motion_r2.values,
                         high_pass = 1/128,
                         t_r = 2.4)
            for x in raw_mat_r2
            )

    # Voxel-internal z-scoring
    raw_mat_r1 = list(scipy.stats.zscore(x, axis=0) for x in raw_mat_r1)
    raw_mat_r2 = list(scipy.stats.zscore(x, axis=0) for x in raw_mat_r2)

    # TR wise z-scoring
    raw_mat_r1 = list(scipy.stats.zscore(x, axis=1) for x in raw_mat_r1)
    raw_mat_r2 = list(scipy.stats.zscore(x, axis=1) for x in raw_mat_r2)

    # Load behavioral runs as concatenated file
    behav = Load_behavioral(bids_dir, sub_id, relate=1, concat=1)

    # Relate TRs to behavioral
    # Data frame holding TRs and associated direction
    labels_raw = pd.DataFrame()
    # trs of functional data (+1 since TRs in behav file are counted
    # from starting 0)
    labels_raw['tr'] = np.arange(raw_mat[0].shape[0]) + 1
    # Add column holding directional values for each TR
    labels_raw['angle_mode'] = float('nan')
    labels_raw['angle_mean'] = float('nan')

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Get directional value for each TR
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # Loop over each TR with directional event (mode and mean)
    for tr_count, tr_id in enumerate(
            np.unique(behav['TR number corrected'])
            ):
        # Index of behavioral data for a certain TR
        index = behav.loc[:,'TR number corrected'] == tr_id
        # Get distribution of raw angles during TR
        angle_dist = np.round(
                behav.loc[index,'direction angle by location'].values,
                2)
        # Get mode and mean of angles during one TR
        mode_angle = scipy.stats.mode(angle_dist).mode[0]
        mean_angle = np.mean(angle_dist)

        # Fill label array with directional events (fold not needed since
        # betas don't contain backwards events we are predicting)
        # Eliminate cases in which corrected TR is beyond the collected TRs
        # (since corrected TRs is +2 TRs that might not have been measured)
        if tr_id <= raw_mat[0].shape[0]:
            index = labels_raw.loc[:,'tr'] == tr_id
            labels_raw.loc[index,'angle_mode'] = mode_angle
            labels_raw.loc[index,'angle_mean'] = np.round(mean_angle, 2)

    # Get label vector that only contains events (eliminate NaN entries)
    index = np.where(~np.isnan(labels_raw.loc[:,'angle_mean']))[0]
    predict_labels = labels_raw.loc[index,:]

    # Add bin and prediction column to label vector
    predict_labels['bin_mode'] = float('nan')
    predict_labels['bin_mean'] = float('nan')
    predict_labels['prediction'] = float('nan')

    # Fill bin columns with correct bin
    predict_labels['bin_mode'] = np.floor(
            predict_labels['angle_mode'] / 60
            ) + 1
    predict_labels['bin_mean'] = np.floor(
            predict_labels['angle_mean'] / 60
            ) + 1

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Mask specific analysis
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # Loop over masks
    for mask_count, mask_id in enumerate(mask_names):

        # Give message to user: Processed mask
        print('\t', mask_id, '\t:', end='')

        # Get current mask (unbuffered and buffered)
        c_mask = mask_mat[:,:,:,mask_count]
        c_mask_b1 = mask_mat_b1[:,:,:,mask_count]
        c_mask_b2 = mask_mat_b2[:,:,:,mask_count]

        # Get current beta_mat for both buffers
        c_beta_mat_b1 = beta_mat_b1[mask_count]
        c_beta_mat_b2 = beta_mat_b2[mask_count]

        # z-score beta mat (whole single beta) for both buffers
        c_beta_mat_b1 = scipy.stats.zscore(c_beta_mat_b1, axis=1)
        c_beta_mat_b2 = scipy.stats.zscore(c_beta_mat_b2, axis=1)

        # Get current concatenated raw_mat (from shared mask)
        c_raw_mat_r1 = raw_mat_r1[mask_count]
        c_raw_mat_r2 = raw_mat_r2[mask_count]

        # Concatenate cleaned runs
        c_raw_mat = np.concatenate([c_raw_mat_r1, c_raw_mat_r2])

        # Eliminate volumes from raw_mat which can't be predicted (since TRs
        # which where in no events)
        index = predict_labels.loc[:,'tr'] - 1
        predict_raw_mat = c_raw_mat[index,:]

        # Allocate df holding accuracy of classification for each fold
        results_accuracy = pd.DataFrame()
        results_accuracy['accuracy_mean'] = float('nan')
        results_accuracy['accuracy_mode'] = float('nan')
        results_accuracy['correlation_mean'] = float('nan')
        results_accuracy['correlation_mode'] = float('nan')

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # Start classification
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        # Give message to user: Classification starts
        print('\tClassification...', end='')

        # Classification of backwards events (no corss valitation needed since
        # backwards events are no in buffers)

        # Get training set and trinaing labels for fold (over concatenated
        # buffers of betas)
        train_set = np.concatenate([c_beta_mat_b1, c_beta_mat_b2])
        train_labels = np.concatenate([labels_beta.loc[:,'angle_bin'].values,
                                       labels_beta.loc[:,'angle_bin'].values])

        # Get testing set and testing labels (over concatenated and cleaned
        # raw_data)
        test_set = predict_raw_mat

        # Labels and bins for mode
        test_labels_mode = predict_labels.loc[:, 'angle_mode'].values
        test_bins_mode = predict_labels.loc[:, 'bin_mode'].values
        # Labels and bins for mean
        test_labels_mean = predict_labels.loc[:, 'angle_mean'].values
        test_bins_mean = predict_labels.loc[:, 'bin_mean'].values

        # Create classifier object
        clf = sklearn.svm.LinearSVC(C=1,
                                    max_iter=10000)

        # Train classifier
        clf.fit(train_set, train_labels)

        # Predict angle_bins
        prediction = clf.predict(test_set)

        # Put prediction into label vector
        predict_labels.loc[:,'prediction'] = prediction

        # Save accuracy for mean and mode by comparing number of correct
        # predictions of bin to overall number of predictions
        # Mean
        results_accuracy.loc[0,'accuracy_mean'] = (
                np.sum(np.equal(prediction, test_bins_mean)) /
                len(prediction)
                )
        # Mode
        results_accuracy.loc[0,'accuracy_mode'] = (
                np.sum(np.equal(prediction, test_bins_mode)) /
                len(prediction)
                )

        # Save correlation between predicted bin and continuous label
        # Mean
        results_accuracy.loc[0,'correlation_mean'] = (
                scipy.stats.pearsonr(prediction, test_labels_mean)[0]
                )
        # Mode
        results_accuracy.loc[0,'correlation_mode'] = (
                scipy.stats.pearsonr(prediction, test_labels_mode)[0]
                )

        # Save label vector with predictions into data frame
        results_prediction = deepcopy(predict_labels)
        results_prediction['sub_id'] = sub_id
        results_prediction['mask'] = mask_id

        # Add subject id and mask to data frame of accuracy
        results_accuracy['sub_id']= sub_id
        results_accuracy['mask']= mask_id

        # Append to results data frame
        clf_prediction = clf_prediction.append(results_prediction)
        clf_accuracy = clf_accuracy.append(results_accuracy)

        # Give message to user: Classification done
        print('Done | ', end='')

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # Start permutation
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        # Give message to user: Permutation starts
        print('Permutation...', end='')

        # Same classification process with permuted labels
        for perm_count in np.arange(n_permutation):

            # Get training set and training labels for fold (over
            # concatenated buffers of betas)
            train_set = np.concatenate([c_beta_mat_b1, c_beta_mat_b2])

            # Shuffle labels inside each fold per buffer
            # Loop over cross validation folds (iterated fold is testing fold)
            train_labels = []
            for fold_count in np.arange(8):
                shuffled_labels = sample(list(np.arange(1,7)), k=6)
                train_labels = np.concatenate([train_labels, shuffled_labels])

            # Get testing set and testing labels (over concatenated and
            # cleaned raw_data) (test labels stay unshuffled)
            test_set = predict_raw_mat
            # Labels and bins for mode
            test_labels_mode = predict_labels.loc[:, 'angle_mode'].values
            test_bins_mode = predict_labels.loc[:, 'bin_mode'].values
            # Labels and bins for mean
            test_labels_mean = predict_labels.loc[:, 'angle_mean'].values
            test_bins_mean = predict_labels.loc[:, 'bin_mean'].values

            # Create classifier object
            clf = sklearn.svm.LinearSVC(C=1,
                                        max_iter=10000)

            # Train classifier
            clf.fit(train_set, train_labels)

            # Predict angle_bins
            prediction = clf.predict(test_set)

            # Put prediction into label vector
            predict_labels.loc[:,'prediction'] = prediction

            # Save accuracy for mean and mode by comparing number of
            # correct predictions of bin to overall number of predictions
            # Mean
            results_accuracy.loc[:,'accuracy_mean'] = (
                    np.sum(np.equal(prediction, test_bins_mean)) /
                    len(prediction)
                    )
            # Mode
            results_accuracy.loc[:,'accuracy_mode'] = (
                    np.sum(np.equal(prediction, test_bins_mode)) /
                    len(prediction)
                    )

            # Save correlation between predicted bin and continuous label
            # Mean
            results_accuracy.loc[:,'correlation_mean'] = (
                    scipy.stats.pearsonr(prediction, test_labels_mean)[0]
                    )
            # Mode
            results_accuracy.loc[:,'correlation_mode'] = (
                    scipy.stats.pearsonr(prediction, test_labels_mode)[0]
                    )

            # Save label vector with predictions into data frame
            results_prediction = deepcopy(predict_labels)
            results_prediction['sub_id'] = sub_id
            results_prediction['mask'] = mask_id
            results_prediction['permutation'] = perm_count

            # Add subject id and mask to data frame of accuracy
            results_accuracy['sub_id']= sub_id
            results_accuracy['mask']= mask_id
            results_accuracy['permutation']= perm_count

            # Append to results data frame
            clf_prediction_permut = clf_prediction_permut.append(
                    results_prediction)
            clf_accuracy_permut = clf_accuracy_permut.append(results_accuracy)

        # Give message to user: Permutation done
        print('Done')

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Save results for subject
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # Sort results DFs and save to .tsv
    # Prediction
    clf_prediction = clf_prediction[['sub_id', 'mask',
                                     'tr',
                                     'angle_mode', 'angle_mean',
                                     'bin_mode', 'bin_mean',
                                     'prediction']]
    path = out_dir + '/' + sub_id + '_clf_prediction.tsv'
    clf_prediction.to_csv(path, sep='\t', index=False)
    # Accuracy
    clf_accuracy = clf_accuracy[['sub_id', 'mask',
                                 'accuracy_mode', 'accuracy_mean',
                                 'correlation_mode', 'correlation_mean']]
    path = out_dir + '/' + sub_id + '_clf_accuracy.tsv'
    clf_accuracy.to_csv(path, sep='\t', index=False)

    # Permutation files
    # Prediction
    clf_prediction_permut = clf_prediction_permut[['sub_id', 'mask',
                                                   'tr',
                                                   'angle_mode',
                                                   'angle_mean',
                                                   'bin_mode', 'bin_mean',
                                                   'prediction',
                                                   'permutation']]
    path = out_dir + '/' + sub_id + '_clf_prediction_permut.tsv'
    clf_prediction_permut.to_csv(path, sep='\t', index=False)
    # Accuracy
    clf_accuracy_permut = clf_accuracy_permut[['sub_id', 'mask',
                                               'accuracy_mode',
                                               'accuracy_mean',
                                               'correlation_mode',
                                               'correlation_mean',
                                               'permutation']]
    path = out_dir + '/' + sub_id + '_clf_accuracy_permut.tsv'
    clf_accuracy_permut.to_csv(path, sep='\t', index=False)

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # Basic plotting
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

#    for mask_count, mask_id in enumerate(mask_names):
#
#        fig = plt.figure(figsize=(8,8))
#        gs = gridspec.GridSpec(2,3, wspace=0.3, hspace=0.2)
#
#        mask_results = clf_prediction.loc[clf_prediction['mask'] == mask_id,:]
#
#        for bin_count, bin_id in enumerate(np.arange(1,7)):
#
#            pred_dist = mask_results.loc[mask_results['bin_mode'] == bin_id,
#                                         'prediction']
#
#            ax = fig.add_subplot(gs[bin_count])
#            ax.hist(pred_dist, bins=np.arange(1,8), density=True, rwidth=0.8)
#            ax.set_xticks(np.arange(1,7)+0.5)
#            ax.set_xticklabels(np.arange(1,7))
#            ax.set_ylim(bottom=0, top=1)
#            ax.set_title('Direction ' + str(bin_id), weight='bold')
#            fig.add_subplot(ax)
#        plt.show()

# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------

''' Decoding for subject depending on input'''

# Set up paths
# BIDS directory (containing betas and raw data)
bids_dir = str(Path.home()) + '/direction_decoding_BIDS'
# Path of repository
repo_dir = str(Path.home() + '/direction_decoding')
# Path to save results to (relative to repo)
save_dir = (repo_dir + '/decoding/backwards_decoding/train_beta_test_raw' +
            '/results')

# Define key variables based on command line input
bids_dir = argv[1]
sub_id = argv[2]
mask_names = argv[3]
# Convert input to list
mask_names = mask_names.strip('[]').split(',')
n_permutation = int(argv[4])
out_dir = argv[5]

# Call decoding function with user inputs
main(bids_dir, sub_id, mask_names, n_permutation, out_dir)


import os
from pathlib import Path
from collections import defaultdict
import torch
import torch.nn as nn
import torchaudio
from torch.utils.data import Dataset as TorchDataset
import numpy as np

from utils.audio import process_wav

# inherits Dataset from PyTorch
class Dataset(TorchDataset):

    def __init__(self, name, split, sample_rate, audio_length, tracks_list, num_segments, audio_proc_dir, mean, std):
        self.name = name
        self.split = split
        self.audio_proc_dir
        self.sample_rate = sample_rate
        self.audio_length = audio_length
        self.tracks_list = tracks_list
        self.num_segments = num_segments
        self.mean = mean
        self.std = std

        print(f"[{self.name} {self.split}]: Loaded {len(self.tracks_list)} audio segments")

    def indexer(self, ids, id2audio_path, id2gt, dataset, onepos=False):
        index = []
        tmp = []
        track_index = defaultdict(list)
        track_idx = 0
        clip_idx = 0
        for clip_id in ids:
            fp = id2audio_path[clip_id]
            label = id2gt[clip_id]

            if dataset == "magnatagatune":
                track_id = "".join(Path(fp).stem.split("-")[:-2])
                if track_id not in tmp:
                    tmp.append(track_id)
                    track_idx += 1
                fp = f"{track_idx}-{clip_id}-{self.sample_rate}.wav"
                fp = os.path.join(self.audio_proc_dir, self.split, fp)
            else:
                track_idx = clip_id
                clip_id = clip_idx
                fp = os.path.join(self.audio_proc_dir, fp)
                clip_idx += 1

            for s in range(self.num_segments):
                index.append([track_idx, clip_id, s, fp, label])
                track_index[track_idx].append([clip_id, s, fp, label])
        return index, track_index

    def loader(self, path):
        audio, sr = process_wav(self.sample_rate, path, False)
        return audio, sr

    def get_audio(self, fp):
        audio, sr = self.loader(fp)
        max_samples = audio.shape[0]
        # if sr != self.sample_rate:
        #     raise Exception("Sample rate is not consistent throughout the dataset")

        if max_samples - self.audio_length <= 0:
            raise Exception("Max samples exceeds number of samples in crop")

        # if np.isnan(audio).any():
        #     raise Exception("Audio contains NaN values")

        return audio

    def __len__(self):
        return len(self.tracks_list)

    def get_full_size_audio(self, fp):
        audio = self.get_audio(fp)

        # split into equally sized tensors of self.audio_length
        audio = torch.from_numpy(audio).reshape(1, -1)
        batch = torch.split(audio, self.audio_length, dim=1)

        # remove last, since it is not a full self.audio_length, and stack
        batch = torch.cat(batch[:-1])

        # reshape to B x 1 x N
        batch = batch.reshape(batch.shape[0], 1, -1)
        return batch

    def normalise_audio(self, audio):
        return (audio - self.mean) / self.std

    def denormalise_audio(self, norm_audio):
        return (norm_audio * self.std) + self.mean

    def sample_audio_by_track_id(self, track_id, batch_size=20):
        """
        Get audio samples based on the track_id (batch_size = num_samples)
        used for plotting the latent representations of different tracks
        """
        batch = torch.zeros(batch_size, 1, self.audio_length)
        for idx in range(batch_size):
            _, _, fp, _ = self.track_index[track_id][0]

            audio = self.get_audio(fp)

            start_idx = idx * self.audio_length

            # too large
            if (start_idx + self.audio_length) > audio.shape[0]:
                return None

            batch[idx, 0, :] = torch.from_numpy(audio[start_idx : start_idx + self.audio_length])
        return batch


from matplotlib import pylab, pyplot as plt
import networkx as nx
import json
import os
import numpy as np
from adjacency import *
from scipy import signal

# SET THESE THREE PARAMS
'''
num:     [0-8] See "sets" variable.

         Set 2 has 3 agents, so you have to give options '20', '21', and '22'
         which correspond to the 1st, 2nd, and 3rd agents respectively.

         Set 7 has 2 agents, so you have to set'70' or '71' accordingly.

cen_num: 0:closeness (num == 0,20,21,22, 9), 1:degree (num == 1 and 6), 2: eigenvector (num == 3,4)

color:   Turn on region coloring
'''
num = 6
cen_num = 0
color = False

# ================================================================================================

agent_num = 0
if 20 <= num <= 30:
    agent_num = num % 20
    num = int(num / 10)

if 40 <= num <= 80:
    agent_num = num % 70
    num = int(num / 10)

sets = ['4',                        # 0
        '12_55_47',                 # 1
        '11_30_18',                 # 2
        '7_30_18_2',                # 3
        '10_30_18',                 # 4
        '13-2019-08-27-22-30-18',   # 5
        '16_30_18',                 # 6
        '9_47_10',                  # 7
        '5_55_47',                  # 8
        '10_55_47']                 # 9
frames = [[0, 60], [47, 80], [[55, 95], [70, 98],[70, 98]],
          [52, 98], [14, 50], [59, 85], [65, 72], [ [9, 35], [50, 98]],
          [58, 90], [10, 98]]
agent_IDs = [983, 2677, [2810, 2958, 2959], 1336, 3494, 1295, 1786, [[2562], [2564]], 1750, 868]
radius = [10, 20, 10, 20, 20, 10, 20, 10, 10, 10]
agent_labels = ['Black Car', 'White Car', 'White Car', 'White Bus',
                'White Truck', 'White Lorry', 'Motorbike', 'Scooter', 'Scooter', 'Motorbike']
centrality_labels = ['Closeness Centrality Value', 'Degree Centrality Value', 'Eigenvector Centrality Value']
thresholds = [[0, 35, 60], [47, 61, 80], [[55, 80, 95], [70, 80, 98], [70, 80, 98]], [52, 73, 98], [14, 40, 50],
              [59, 75, 85], [65, 68, 72], [ [9, 18, 35], [50, 75, 98]], [58, 80, 90], [24, 36]]

video_set = sets[num]
frame = frames[num] if (num != 2 and num != 7) else frames[num][agent_num]
agent_ID = agent_IDs[num]
rad = radius[num]
agent_label = agent_labels[num]
centrality_label = centrality_labels[cen_num]
x_lims = thresholds[num] if (num != 2 and num != 7) else thresholds[num][agent_num]

# filenames = os.listdir('data/')

# READ CSV FILE PANDAS
filepath = 'data/set' + str(video_set) + '_annotations_utm.json'

with open(filepath) as json_file:
    sample_annotation = json.load(json_file)

dataset_id = 1

to_list = []

for each_frame in sample_annotation:

    fr_id = each_frame['id']
    each_annotation = each_frame['annotations']
    for each_object in each_annotation:
        obj_id = each_object['classId']
        x = each_object['geometry']['position']['x']
        y = each_object['geometry']['position']['y']
        to_list.append([fr_id, obj_id, x, y, dataset_id])

to_array = np.asarray(to_list)
obj_IDs = np.unique(to_array[:, 1]).astype(int)
agent_idx = list(obj_IDs).index(agent_ID[agent_num]) if (num == 2 or num == 7) else list(obj_IDs).index(agent_ID)
degree_mat, adj_mats = generate_adjacency(to_array, rad)
weave_list = []
weave_list2 = []
weave_list3 = []
for fr, item in enumerate(adj_mats):
    G = nx.from_numpy_array(item['adj_matrix'])
    if cen_num == 0:
        g = nx.closeness_centrality(G)
    elif cen_num == 1:
        g = nx.degree_centrality(G)
    else:
        g = nx.eigenvector_centrality(G, max_iter=10000)
    # eg = nx.eigenvector_centrality(G)
    # print(fr,g[agent_idx])
    # if fr < 60:
    if frame[0] <= fr <= frame[1]:
        weave_list.append(g[0])  # num/id- 0,0, 1/1, 2/0, 3/0, 4/0
        weave_list2.append(g[agent_idx])
        # weave_list2.append(cg[7])
    # weave_list3.append(cg[8])

buffer2 = 7 if len(weave_list2) % 2 == 0 else 8
buffer = 7 if len(weave_list) % 2 == 0 else 8
if num != 6:
    weave_list2 = signal.savgol_filter(weave_list2, len(weave_list2)-buffer2, 3)
    # weave_list = signal.savgol_filter(weave_list, len(weave_list)-buffer, 3)

cmaps = ['#0038ff', '#00a4b2', '#4c6fb8', '#2c5167', '#9fd9ea']
LineThick = 5
FontSize = 40

fig, ax = plt.subplots(figsize=(11.0, 8.0))
x = np.arange(frame[0], frame[1] + 1)
y = weave_list2
# x = np.arange(frame[0], frame[1])
# y = weave_list2[2:] - 2 * weave_list2[:-1]
if len(agent_label.split()) > 1:
    agent_color = agent_label.split()[0]
else:
    agent_color = 'Gold'
ax.plot(x, y, linewidth=LineThick, label=agent_label, color='k')
if num == 0:
    ax.plot(x, weave_list, linewidth=LineThick, color='k')
    # ax.plot(range(frame[0], frame[1]+1), weave_list, linewidth=LineThick, label='Red Car', color='Tomato')
if color:
    for i in range(0, len(x_lims) - 1):
        ax.fill_between([x_lims[i], x_lims[i + 1]],
                        np.max(y[x_lims[i]-frame[0]:x_lims[i + 1]-frame[0]+1]) + 0.002,
                        np.min(y[x_lims[i]-frame[0]:x_lims[i + 1]-frame[0]+1]) - 0.002,
                        facecolor=cmaps[i], alpha=0.6, interpolate=True)
        # ax.fill_between([25, 35], np.max(y),np.min(y), facecolor='red', alpha=0.2, interpolate=True)
        # ax.fill_between([75, 85], np.max(y),np.min(y), facecolor='red', alpha=0.2, interpolate=True)
plt.grid(True)
plt.xlabel('Frame Number', fontsize=FontSize)
plt.ylabel(centrality_label.split()[0], fontsize=FontSize)
for tick in ax.xaxis.get_major_ticks():
    tick.label.set_fontsize(FontSize - 7)
for tick in ax.yaxis.get_major_ticks():
    tick.label.set_fontsize(FontSize - 7)
# Turn off tick labels
# ax.set_yticklabels([])
# legend = plt.legend(loc='lower left', bbox_to_anchor=(0, 1.01), ncol=2,
#                     borderaxespad=0, fontsize=FontSize - 5, fancybox=True,
#                     facecolor='green', framealpha=0.4)
# frame = legend.get_frame()
# frame.set_facecolor('green')
# frame.set_edgecolor('red')
# plt.savefig('images/' + video_set + '_' + agent_label + '.png', bbox_inches='tight')
# fig2, ax2 = plt.subplots(figsize=(15.0, 12.0))
# line, = ax2.plot(x, y,linewidth=LineThick, color='k')
# plt.grid(True)
# # plt.xlabel('Frame Number', fontsize=FontSize)
# # plt.ylabel(centrality_label.split()[0], fontsize=FontSize)
# for tick in ax2.xaxis.get_major_ticks():
#     tick.label.set_fontsize(FontSize - 7)
# for tick in ax2.yaxis.get_major_ticks():
#     tick.label.set_fontsize(FontSize - 7)
# y_init = list([0]*len(y))

# for n in range(len(x)):
#     line.set_data(x[:n], y[:n])
#     # ax2.axis([0, 100, 0, ])
#     fig.canvas.draw()
#     plt.savefig('video_materials/' + video_set + '_' + '{}.png'.format(n), bbox_inches='tight')

# plt.plot(range(frame[0], frame[1]+1), weave_list2, linewidth= LineThick, label=agent_label )
# if num==0:
#     plt.plot(range(frame[0], frame[1]+1), weave_list, linewidth= LineThick,label="Scooter" )
# plt.grid(True)
# plt.xlabel('Time (Frame Number)', fontsize=FontSize)
# plt.ylabel(centrality_label, fontsize=FontSize)
# plt.legend(loc='upper left', fontsize=FontSize)
plt.show()


# -*- coding: utf-8 -*-
import os
import re
import tempfile
from test.data import (
    bob,
    cheese,
    context0,
    context1,
    context2,
    hates,
    likes,
    michel,
    pizza,
    tarek,
)

import pytest
from rdflib import RDF, RDFS, BNode, Literal, URIRef, Variable
from rdflib.graph import Dataset, QuotedGraph

storename = "SQLiteStore"
path = tempfile.mktemp(prefix="test", dir="/tmp")

graphuri = URIRef("urn:example:graph")
othergraphuri = URIRef("urn:example:othergraph")

implies = URIRef("http://www.w3.org/2000/10/swap/log#implies")

test_n3 = """\
@prefix rdf:  <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix : <http://test/> .
{:a :b :c;a :foo} => {:a :d :c,?y} .
_:foo a rdfs:Class .
:a :d :c .
"""

def expunge_path():
    if path is not None:
        if os.path.exists(path):
            if os.path.isdir(path):
                for f in os.listdir(path):
                    os.unlink(path + "/" + f)
                os.rmdir(path)
            elif len(path.split(":")) == 1:
                os.unlink(path)
            else:
                os.remove(path)

@pytest.fixture(scope="function")
def dataset():

    expunge_path()
    graph = Dataset(store=storename, identifier=context0)
    graph.open(path, create=True)

    yield graph

    graph.close()

    expunge_path()

def test_simple_graph(dataset):
    graph = dataset.get_context(graphuri)
    graph.add((tarek, likes, pizza))
    graph.add((bob, likes, pizza))
    graph.add((bob, likes, cheese))

    g2 = dataset.get_context(othergraphuri)
    g2.add((michel, likes, pizza))

    assert len(graph) == 3, "graph contains 3 triples"
    assert len(g2) == 1, "other graph contains 1 triple"

    r = graph.query("SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }")
    assert len(list(r)) == 2, "two people like pizza"

    r = graph.triples((None, likes, pizza))
    assert len(list(r)) == 2, "two people like pizza"

    # Test initBindings
    r = graph.query(
        "SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }",
        initBindings={"s": tarek},
    )
    assert len(list(r)) == 1, "i was asking only about tarek"

    r = graph.triples((tarek, likes, pizza))
    assert len(list(r)) == 1, "i was asking only about tarek"

    r = graph.triples((tarek, likes, cheese))
    assert len(list(r)) == 0, "tarek doesn't like cheese"

    g2.add((tarek, likes, pizza))
    graph.remove((tarek, likes, pizza))
    r = graph.query("SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }")

def test_dataset_default(dataset):
    dataset.default_union = True

    subgraph1 = dataset.get_context(context1)

    subgraph1.add((tarek, likes, pizza))

    subgraph2 = dataset.get_context(context2)
    subgraph2.add((bob, likes, pizza))

    subgraph1.add((tarek, hates, cheese))

    assert len(subgraph1) == 2, "graph contains 2 triples"

    # the following are actually bad tests as they depend on your endpoint,
    # as pointed out in the sparqlstore.py code:
    #
    # # For Datasets, reading is done from the "default graph" Exactly
    # # what this means depends on your endpoint, because SPARQL does not offer a
    # # simple way to query the union of all graphs as it would be expected for a
    # # ConjuntiveGraph.
    # #
    # # Fuseki/TDB has a flag for specifying that the default graph
    # # is the union of all graphs (tdb:unionDefaultGraph in the Fuseki config).
    assert (
        len(dataset) == 3
    ), f"default union graph should contain three triples but contains {list(dataset)}:\n"

    r = dataset.query("SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }")

    assert len(list(r)) == 2, f"two people should like pizza, not {len(list(r))}"

    r = dataset.query(
        "SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }",
        initBindings={"s": tarek},
    )
    assert len(list(r)) == 1, "i was asking only about tarek"

    r = dataset.triples((tarek, likes, pizza))
    assert len(list(r)) == 1, "i was asking only about tarek"

    r = dataset.triples((tarek, likes, cheese))
    assert len(list(r)) == 0, "tarek doesn't like cheese"

    subgraph2.remove((bob, likes, pizza))

    r = dataset.query("SELECT * WHERE { ?s <urn:example:likes> <urn:example:pizza> . }")
    assert len(list(r)) == 1, "only tarek likes pizza"

def test_update(dataset):
    dataset.update(
        "INSERT DATA { GRAPH <urn:example:graph> { <urn:example:michel> <urn:example:likes> <urn:example:pizza> . } }"
    )

    graph = dataset.get_context(graphuri)
    assert len(graph) == 1, "graph contains 1 triple"

def test_update_with_initns(dataset):
    dataset.update(
        "INSERT DATA { GRAPH ns:graph { ns:michel ns:likes ns:pizza . } }",
        initNs={"ns": URIRef("urn:example:")},
    )

    graph = dataset.get_context(graphuri)
    assert set(graph.triples((None, None, None))) == set(
        [(michel, likes, pizza)]
    ), "only michel likes pizza"

def test_update_with_initbindings(dataset):
    dataset.update(
        "INSERT { GRAPH <urn:example:graph> { ?a ?b ?c . } } WherE { }",
        initBindings={
            "a": michel,
            "b": likes,
            "c": pizza,
        },
    )

    graph = dataset.get_context(graphuri)
    assert set(graph.triples((None, None, None))) == set(
        [(michel, likes, pizza)]
    ), "only michel likes pizza"

def test_multiple_update_with_initbindings(dataset):
    dataset.update(
        "INSERT { GRAPH <urn:example:graph> { ?a ?b ?c . } } WHERE { };"
        "INSERT { GRAPH <urn:example:graph> { ?d ?b ?c . } } WHERE { }",
        initBindings={
            "a": michel,
            "b": likes,
            "c": pizza,
            "d": bob,
        },
    )

    graph = dataset.get_context(graphuri)
    assert set(graph.triples((None, None, None))) == set(
        [(michel, likes, pizza), (bob, likes, pizza)]
    ), "michel and bob like pizza"

def test_named_graph_r3_update(dataset):
    graph = dataset.get_context(graphuri)

    # Strings with unbalanced curly braces
    tricky_strs = ["With an unbalanced curly brace %s " % brace for brace in ["{", "}"]]
    for tricky_str in tricky_strs:
        r3 = (
            """INSERT { ?b <urn:example:says> "%s" }
        WHERE { ?b <urn:example:likes> <urn:example:pizza>} """
            % tricky_str
        )
        graph.update(r3)

def test_named_graph_update(dataset):
    graph = dataset.get_context(graphuri)
    r1 = "INSERT DATA { <urn:example:michel> <urn:example:likes> <urn:example:pizza> }"
    graph.update(r1)
    assert set(graph.triples((None, None, None))) == set(
        [(michel, likes, pizza)]
    ), "only michel likes pizza"

    r2 = (
        "DELETE { <urn:example:michel> <urn:example:likes> <urn:example:pizza> } "
        + "INSERT { <urn:example:bob> <urn:example:likes> <urn:example:pizza> } WHERE {}"
    )
    graph.update(r2)
    assert set(graph.triples((None, None, None))) == set(
        [(bob, likes, pizza)]
    ), "only bob likes pizza"
    says = URIRef("urn:example:says")

    # Strings with unbalanced curly braces
    tricky_strs = ["With an unbalanced curly brace %s " % brace for brace in ["{", "}"]]
    for tricky_str in tricky_strs:
        r3 = (
            """INSERT { ?b <urn:example:says> "%s" }
        WHERE { ?b <urn:example:likes> <urn:example:pizza>} """
            % tricky_str
        )
        graph.update(r3)

    values = set()
    for v in graph.objects(bob, says):
        values.add(str(v))
    assert values == set(tricky_strs)

# @pytest.mark.xfail(reason="Failure in SQL query string handling")
def test_named_graph_update_complicated_strings(dataset):
    graph = dataset.get_context(graphuri)
    says = URIRef("urn:example:says")
    # Complicated Strings
    r4strings = []
    r4strings.append(r'''"1: adfk { ' \\\" \" { "''')
    r4strings.append(r'''"2: adfk } <foo> #éï \\"''')

    r4strings.append(r"""'3: adfk { " \\\' \' { '""")
    r4strings.append(r"""'4: adfk } <foo> #éï \\'""")

    r4strings.append(r'''"""5: adfk { ' \\\" \" { """''')
    r4strings.append(r'''"""6: adfk } <foo> #éï \\"""''')
    r4strings.append('"""7: ad adsfj \n { \n sadfj"""')

    r4strings.append(r"""'''8: adfk { " \\\' \' { '''""")
    r4strings.append(r"""'''9: adfk } <foo> #éï \\'''""")
    r4strings.append("'''10: ad adsfj \n { \n sadfj'''")

    r4 = "\n".join(
        [
            "INSERT DATA { <urn:example:michel> <urn:example:says> %s } ;" % s
            for s in r4strings
        ]
    )
    graph.update(r4)
    values = set()
    for v in sorted(graph.objects(michel, says)):
        values.add(str(v))
    assert values == set(
        [
            re.sub(
                r"\\(.)",
                r"\1",
                re.sub(r"^'''|'''$|^'|'$|" + r'^"""|"""$|^"|"$', r"", s),
            )
            for s in r4strings
        ]
    )

# @pytest.mark.xfail(reason="Failure in SQL query string handling")
def test_named_graph_update_iri_containing_octothorpe(dataset):
    graph = dataset.get_context(graphuri)

    # IRI Containing ' or #
    # The fragment identifier must not be misinterpreted as a comment
    # (commenting out the end of the block).
    # The ' must not be interpreted as the start of a string, causing the }
    # in the literal to be identified as the end of the block.
    r5 = """INSERT DATA { <urn:example:michel> <urn:example:hates> <urn:example:foo'bar?baz;a=1&b=2#fragment>, "'}" }"""

    graph.update(r5)
    values = set()
    for v in graph.objects(michel, hates):
        values.add(str(v))
    assert values == set(["urn:example:foo'bar?baz;a=1&b=2#fragment", "'}"])

    # Comments
    r6 = """
        INSERT DATA {
            <urn:example:bob> <urn:example:hates> <urn:example:bob> . # No closing brace: }
            <urn:example:bob> <urn:example:hates> <urn:example:michel>.
        }
    #Final { } comment"""

    graph.update(r6)
    values = set()
    for v in graph.objects(bob, hates):
        values.add(v)
    assert values == set([bob, michel])

def test_named_graph_update_with_initbindings(dataset):
    graph = dataset.get_context(graphuri)
    r = "INSERT { ?a ?b ?c } WHERE {}"
    graph.update(r, initBindings={"a": michel, "b": likes, "c": pizza})
    assert set(graph.triples((None, None, None))) == set(
        [(michel, likes, pizza)]
    ), "only michel likes pizza"

def test_empty_literal(dataset):
    # test for https://github.com/RDFLib/rdflib/issues/457
    # also see test_issue457.py which is sparql store independent!
    graph = dataset.get_context(graphuri)
    graph.add(
        (
            URIRef("http://example.com/s"),
            URIRef("http://example.com/p"),
            Literal(""),
        )
    )

    o = tuple(graph)[0][2]
    assert Literal("") == o, repr(o)

def test_n3_store(dataset):
    dataset.default_union = True
    dataset.parse(data=test_n3, format="n3")
    formula_a = BNode()
    formula_b = BNode()
    for s, p, o in dataset.triples((None, implies, None)):
        formula_a = s
        formula_b = o

    assert type(formula_a) == QuotedGraph and type(formula_b) == QuotedGraph
    a = URIRef("http://test/a")
    b = URIRef("http://test/b")
    c = URIRef("http://test/c")
    d = URIRef("http://test/d")
    v = Variable("y")

    universe = Dataset(dataset.store)

    # test formula as terms
    assert len(list(universe.triples((formula_a, implies, formula_b)))) == 1

    # test variable as term and variable roundtrip
    assert len(list(formula_b.triples((None, None, v)))) == 1
    for s, p, o in formula_b.triples((None, d, None)):
        if o != c:
            assert isinstance(o, Variable)
            assert o == v
    s = list(universe.subjects(RDF.type, RDFS.Class))[0]
    assert isinstance(s, BNode)
    assert len(list(universe.triples((None, implies, None)))) == 1
    assert len(list(universe.triples((None, RDF.type, None)))) == 1
    assert len(list(formula_a.triples((None, RDF.type, None)))) == 1
    assert len(list(formula_a.triples((None, None, None)))) == 2
    assert len(list(formula_b.triples((None, None, None)))) == 2
    assert len(list(universe.triples((None, None, None)))) == 3
    assert len(list(formula_b.triples((None, URIRef("http://test/d"), None)))) == 2
    assert len(list(universe.triples((None, URIRef("http://test/d"), None)))) == 1

    # context tests
    # test contexts with triple argument
    assert len(list(universe.contexts((a, d, c)))) == 1

    # Remove test cases
    universe.remove((None, implies, None))
    assert len(list(universe.triples((None, implies, None)))) == 0
    assert len(list(formula_a.triples((None, None, None)))) == 2
    assert len(list(formula_b.triples((None, None, None)))) == 2

    formula_a.remove((None, b, None))
    assert len(list(formula_a.triples((None, None, None)))) == 1
    formula_a.remove((None, RDF.type, None))
    assert len(list(formula_a.triples((None, None, None)))) == 0

    universe.remove((None, RDF.type, RDFS.Class))

    # remove_context tests
    universe.remove_graph(formula_b)
    assert len(list(universe.triples((None, RDF.type, None)))) == 0
    assert len(universe) == 1
    assert len(formula_b) == 0

    universe.remove((None, None, None))
    assert len(universe) == 0

    dataset.close()
    dataset.store.destroy(path)

xmltestdoc = """<?xml version="1.0" encoding="UTF-8"?>
<rdf:RDF
   xmlns="http://example.org/"
   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
>
  <rdf:Description rdf:about="http://example.org/a">
    <b rdf:resource="http://example.org/c"/>
  </rdf:Description>¬
</rdf:RDF>
"""

n3testdoc = """@prefix : <http://example.org/> .

:a :b :c .
"""

nttestdoc = "<http://example.org/a> <http://example.org/b> <http://example.org/c> .\n"


#!/usr/bin/env python
import datetime
import datetime
import numpy as np
import os
from PIL import Image, ImageFilter, ImageDraw, ImageFont
# import time
from bpassW import *
# from matplotlib import pyplot as plt
# from math import pi
from decimal import Decimal
from pkfndW2 import *
from skimage.draw import ellipse

# camera libraries
#from gpiozero import LED
#from picamera import PiCamera
#import io

def format_e(n):
	a = '%E' % n
	return a.split('E')[0].rstrip('0').rstrip('.') + 'E' + a.split('E')[1]

def kukicounter(imagefile):
	np.set_printoptions(precision=5,suppress=True)
	# ## take photo
	# led = LED(17)
	# led.on()
	# print ("LED on")
	# body = camera.get_frame()
	# now = datetime.datetime.now()
	# dstfilename = "Kyu" + now.strftime("%Y-%m-%d %Hh%Mm")
	# with open(dstfilename,'wb') as f:
	# 	f.write(body)
	# 	f.close()
	# print("LED off")
	# led.off()
	#imagefile = 'C:/Users/kuki/Downloads/071910am/1b01.jpeg'
	## pre-processing
	a = Image.open(imagefile) # open image
	ag = a.convert(mode="L") # mode=L (8bit, black and white) aka greyscale
	scf=0.5
	resize = tuple(int(i*scf) for i in ag.size)
	ag = ag.resize(resize, Image.ANTIALIAS)
	a = a.resize(resize, Image.ANTIALIAS)
	ag = np.asarray(ag)

	bg1 = np.mean(ag)
	# mean filter: subtract mean of all pixels from each pixel
	# noise-reduction
	imbr0 = ag-bg1
	imbr = np.abs(imbr0)
	#ag=Image.fromarray(ag)
	#median = ag.filter(ImageFilter.MedianFilter)
	#ag=np.asarray(ag)
	## cell dectection
	b0 = bpassW(imbr,lnoise=1,lobject=3) #put median for median filter

	var1=13
	ibg = np.mean(b0)+var1*np.std(b0)

	var2=20
	pk = pkfndW2(b0,ibg,var2)

	pk5=pk

	cc=np.zeros(len(pk5))
	cc[pk5.T[0]>0]=True
	cc[pk5.T[1]>0]=True
	pks=pk5

	no_cells = len(pks)*2

	print ('detected {:3.0f} object inbound out of total {:3.0f}'.format(np.sum(cc),len(cc)))

	# draw dot on top of cells
	a_draw = ImageDraw.Draw(a)
	for centroid in pks:
		y = centroid[0]
		x = centroid[1]
		e=[x-1,y-1,x+1,y+1]
		a_draw.ellipse(e,fill='rgb(100%,0%,0%)')
	del a_draw

	# area_px = 'imagesize'
	# px_size = 7.45 #um/px
	# area_mm = area_px*(px_size*0.001)**2
	# chamber_height = 0.1 #100nm
	# chamber_vol = (area_mm*chamber_height)/1000
	# density = format_e(Decimal(str(no_cells/chamber_vol)))
	# densitye = Decimal(str(no_cells/chamber_vol))
	# print ('cell density-- {:.2E}cells/ml'.format(densitye))

	# get a font
	#fnt = ImageFont.load_default()
	# get a drawing context
	#d = ImageDraw.Draw(a)
	# draw text
	#d.text((10,10), 'cell density-- {:.2E}cells/ml'.format(densitye), font=fnt, fill=(255,255,255))
	# draw text, full opacity
	#d.text((10,60), "World", font=fnt, fill=(255,255,255))
	file,ext=os.path.splitext(os.path.basename(imagefile))

	# create destination folder
	desktop = os.path.join(os.environ["USERPROFILE"], "Desktop")
	dstfolder = os.path.join('C:/Users/kuki/Downloads/071910am','count_results')
	if not os.path.exists(dstfolder): os.mkdir(dstfolder)
	# prevent overwriting counted image
	if not os.path.exists(file+'_counted.jpg'):
		a.save(os.path.join(dstfolder,file+'_counted.jpg'),format="JPEG")
	return no_cells

fol='C:/Users/kuki/Downloads/071910am'
ims = [os.path.join(fol,_) for _ in os.listdir(fol) if _.endswith('jpeg')]
for im in ims:
	no_cells = kukicounter(im)
print(no_cells)