from celery import shared_task
from django.core.mail import send_mail

@shared_task
def send_email(receive):

    subject = 'yes'
    message = 'really'
    form_email = 'rongjiawei1204@163.com'
    recipient_list = (receive,)
    send_mail(subject=subject, message=message,from_email=form_email,recipient_list=recipient_list)


import pandas as pd
import os
import sys
import argparse

def one_year_csv_writer(this_year, all_data, output):
    """
    Writes a csv file for data from a given year.

    this_year --- year for which data is extracted
    all_data --- DataFrame with multi-year data
    """

    # Select data for the year
    surveys_year = all_data[all_data.year == this_year]

    # Write the new DataFrame to a csv file
    filename = output + str(this_year) + '.csv'
    surveys_year.to_csv(filename)

def output_test(test_dir):
    """
    Check to see if an output directory exists.
    If it exists, tell us.
    If it doesn't, create one, and tell us.
    """
    test_split = test_dir.split('/')
    check_directory = test_split[0] + '/' + test_split[1] + '/'
    if test_split[2] in os.listdir(check_directory):
       print('Processed directory exists')
    else:
#       output_directory = '/'.join(test_split[0:-1])
       output_directory = test_split[0] + '/' + test_split[1] + '/' + test_split[2]
       print(output_directory)
       os.mkdir(output_directory)
       print('Processed directory created')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.addArgument("--dataset", help = "CSV containing data for multiple years.")
    parser.addArgument("--output", help = "Where do you want to write output?")
    parser.addArgument("--year", help = "Which year do you want to analyze?")
    args = parser.parse_args()
    if args.dataset:
        surveys_df = args.dataset
    if args.output:
        test_dir = args.output
    if args.year:
        this_year = args.year
    surveys_df = pd.read_csv(surveys_df)
    output_test(test_dir)
    one_year_csv_writer(all_data = surveys_df, this_year = data_year, output = test_dir)
    print("Victory.")


from scrapy import Request
from scrapy.spiders import Spider
from spider_meizitu.items import SpiderMeizituItem
import re

class MeizituSpider(Spider):
    name = 'meizitu'

    start_urls = {
        'http://www.meizitu.com/a/sexy.html',
    }

    def parse(self, response):
        self.logger.info('Parse function called on %s', response.url)
        meizi_pic_lists = response.xpath('//ul[@class="wp-list clearfix"]/li')
        for meizi_item in meizi_pic_lists:
            meizi_item_url = meizi_item.xpath('.//h3[@class="tit"]/a/@href').extract()[0]
            # print('===== 当前爬取页面共有图片%s组，正在抓取第%s组图片，页面链接:: %s =====' % (len(meizi_pic_lists), i + 1, meizi_item_url))
            yield Request(meizi_item_url, callback=self.parse_meizi_pic)

        # next_url = re.findall('下一页', response.xpath('//*[@id="wp_page_numbers"]').extract()[0])
        # if next_url:
        #     # next_url = 'http://www.meizitu.com/a/' + next_url[0]
        #     # print('========== Request Next Url :: %s ==========' % next_url)
        #     next_url = 'http://www.meizitu.com/a/' + next_url[0]
        #     yield response.follow(next_url, callback=self.parse)

    def parse_meizi_pic(self, response):
        self.logger.info('Parse function called on %s', response.url)
        item = SpiderMeizituItem()
        item['group'] = response.xpath("//div[@class='metaRight']/h2/a/text()").extract()[0]
        meizitu_pics = response.xpath('//div[@id="picture"]/p/img')
        for meizitu_pic in meizitu_pics:
            item['name'] = meizitu_pic.xpath('.//@alt').extract()[0]
            item['url'] = meizitu_pic.xpath('.//@src').extract()[0]
            # print('===== 当前页面共有图片%s张，正在抓取第%s张图片，图片链接:: %s =====' % (len(meizitu_pics), i + 1, item['image_urls']))
            yield item


import sys
from importlib import import_module
import click

from orun.utils import reraise
from orun.apps import apps
from orun.core.management.color import no_style
from orun.core.management.sql import emit_post_migrate_signal, sql_flush
from orun.db import DEFAULT_DB_ALIAS, connections #, transaction

@click.command('flush')
@click.option('--database', '-db')
@click.option('--verbosity', '-v')
@click.option('--interactive', '-i')
@click.option('--reset-sequences', '-r')
@click.option('--allow-cascade', '-c')
@click.option('--inhibit-post-migrate', '-ipm')
def command(self, database, verbosity, interactive, reset_sequences, allow_cascade, inhibit_post_migrate):
    connection = connections[database]

    self.style = no_style()

    # Import the 'management' module within each installed app, to register
    # dispatcher events.
    for app_config in apps.get_app_configs():
        try:
            import_module('.management', app_config.name)
        except ImportError:
            pass

    sql_list = sql_flush(self.style, connection, only_orun=True,
                         reset_sequences=reset_sequences,
                         allow_cascade=allow_cascade)

    if interactive:
        confirm = input("""You have requested a flush of the database.
This will IRREVERSIBLY DESTROY all data currently in the %r database,
and return each table to an empty state.
Are you sure you want to do this?

Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME'])
    else:
        confirm = 'yes'

    if confirm == 'yes':
        try:
            with transaction.atomic(using=database,
                                    savepoint=connection.features.can_rollback_ddl):
                with connection.cursor() as cursor:
                    for sql in sql_list:
                        cursor.execute(sql)
        except Exception as e:
            new_msg = (
                "Database %s couldn't be flushed. Possible reasons:\n"
                "  * The database isn't running or isn't configured correctly.\n"
                "  * At least one of the expected database tables doesn't exist.\n"
                "  * The SQL was invalid.\n"
                "Hint: Look at the output of 'orun-admin sqlflush'. "
                "That's the SQL this command wasn't able to run.\n"
                "The full error: %s") % (connection.settings_dict['NAME'], e)
            reraise(click.UsageError, click.UsageError(new_msg), sys.exc_info()[2])

        # Empty sql_list may signify an empty database and post_migrate would then crash
        if sql_list and not inhibit_post_migrate:
            # Emit the post migrate signal. This allows individual applications to
            # respond as if the database had been migrated from scratch.
            emit_post_migrate_signal(verbosity, interactive, database)
    else:
        self.stdout.write("Flush cancelled.\n")


# Copyright 2019-2021 Wingify Software Pvt. Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Module for making requests, uses requests internally """

import requests

class Connection:
    """ Connection class to provide SDK with network connectivity interfaces """

    def __init__(self):
        """ Initializes connection class with requests session object"""
        self.session = requests.Session()

    def get(self, url, params=None):
        """ Get method, it wraps upon requests' get method.
        Args:
            url (str): Unique resource locator
            params (dict): Parameters to be passed
        Returns:
            dict : Status code and Response text
        """
        try:
            resp = self.session.get(url, params=params)
            return {"status_code": resp.status_code, "text": resp.text}
        except Exception:
            return {"status_code": None, "text": ""}

    def post(self, url, params=None, data=None, headers=None):
        """ Post method, it wraps upon requests' post method.
        Args:
            url (str): Unique resource locator
            params (dict): Parameters to be passed
            data (dict): Json data to be passed
            headers (dict): Headers for request
        Returns:
            dict : Status code and Response text
        """
        try:
            resp = self.session.post(url, params=params, json=data, headers=headers)

            return {"status_code": resp.status_code, "text": resp.text}
        except Exception:
            return {"status_code": None, "text": ""}


from django.shortcuts import render, get_object_or_404
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.auth.models import User

from .models import Post, Category
from .forms import CatTransferForm

def category(request, slug=None):
    if slug:
        instance = get_object_or_404(Category, slug=slug)
        all_posts = Post.published_objects.filter(category=instance)
    else:
        instance = None
        all_posts = Post.published_objects.all()

    ctx = {'category': instance, 'posts': all_posts}
    return render(request, 'category.html', ctx)

def post(request, year, month, slug):
    article = get_object_or_404(Post, publish__year=year,
                                publish__month=month, slug=slug)
    ctx = {'article': article}
    return render(request, 'article.html', ctx)

def user_posts(request, userid):
    user = get_object_or_404(User, id=userid)
    all_posts = Post.objects.filter(author=user, publish__isnull=False)

    ctx = {'author': user, 'posts': all_posts}
    return render(request, 'category.html', ctx)

@staff_member_required
def transfer_posts_tool(request):
    if request.method == 'POST':
        form = CatTransferForm(request.POST)
        if form.is_valid():
            Post.objects.filter(category__in=form.cleaned_data['from_cats']).update(
                category=form.cleaned_data['to_cat'])

    else:
        form = CatTransferForm()

    ctx = {
        'form': form,
    }
    return render(request, 'transfer_tool.html', ctx)


from websocket import WebSocketApp
from threading import Thread
import json
import time

def on_message(ws, msg):
    msg = json.loads(msg.decode('utf-8'))
    print(msg)

def on_error(ws, msg):
    print(msg)

def on_close(ws):
    print('close')

def on_open(ws):
    def run(*args):
        request = '[{"ticket":""},{"type":"ticker","codes":["KRW-BTC"]}]'
        ws.send(request)
        time.sleep(5)
        ws.close

    th = Thread(target=run, daemon=True)
    th.start()

if __name__ == "__main__":
    ws = WebSocketApp("wss://api.upbit.com/websocket/v1",
                      on_message=on_message,
                      on_error=on_error,
                      on_close=on_close,
                      on_open=on_open)
    ws.run_forever()


from nltk import *
from nltk.corpus import stopwords
from nltk.tokenize import RegexpTokenizer
from stop_words import get_stop_words
from nltk.stem.porter import PorterStemmer
from gensim import corpora, models
import gensim
import math
import PyPDF2
import timeit
import spacy

nlp = spacy.load('en')

# Initiate List for NER
ORG = []
PERSON = []
GPE_LOCATION = []
MONEY = []
TIME = []
EVENT = []
OTHER = []

question_list = []
text = """
Bertie Steffink, nephew of the aforementioned Luke, had early in life adopted the profession of
ne'er-do-weel; his father had been something of the kind before him. At the age of eighteen Bertie
had commenced that round of visits to our Colonial possessions, so seemly and desirable in the case
of a Prince of the Blood, so suggestive of insincerity in a young man of the middle-class."""

# Replace the tokens to exclude the NER tokens
# Not including the NER taggings in tokenization

# SYSTEM CODE
# PREPROCESSING
tokenized_lda = word_tokenize(text)

# Tokenized by sentence for LDA
lda_sentence_tokens = []
sentence = ""
i = 0;
for word in tokenized_lda:
    sentence += " " + word
    if (word == '.'):
        lda_sentence_tokens.append(sentence)
        # printing every sentence in a text
        sentence = ""

# LDA
# Preprocess
tokenizer = RegexpTokenizer(r'\w+')

# create English stop words list
en_stop = get_stop_words('english')

# Create p_stemmer of class PorterStemmer
p_stemmer = PorterStemmer()

# compile sample documents into a list
doc_set = lda_sentence_tokens

# list for tokenized documents in loop
texts = []

# loop through document list
for i in doc_set:
    # clean and tokenize document string
    doc = nlp(i)

    for ent in doc.ents:
        if ent.label_ == 'ORG':
            ORG.append(ent.text)
            print(ent.text, ent.label_)
        elif ent.label_ == 'PERSON':
            PERSON.append(ent.text)
            print(ent.text, ent.label_)
        elif ent.label_ == 'GPE':
            GPE_LOCATION.append(ent.text)
            print(ent.text, ent.label_)
        elif ent.label_ == 'MONEY':
            MONEY.append(ent.text)
            print(ent.text, ent.label_)
        elif ent.label_ == 'TIME':
            TIME.append(ent.text)
            print(ent.text, ent.label_)
        elif ent.label_ == 'EVENT':
            EVENT.append(ent.text)
            print(ent.text, ent.label_)
        else:
            OTHER.append(ent.text)
            print(ent.text, ent.label_)

NER = ORG + PERSON + GPE_LOCATION + MONEY + TIME + EVENT + OTHER
if " " in NER:
    NER.remove(" ")
print(NER)

for word in NER:

    for sentence in doc_set:

        if "________" in sentence:
            break
        elif word in sentence:
            question_list.append(sentence.replace(word, "________"))
index = 1
for question in question_list:
    print(str(index) + "] " +question)
    index += 1
"""
for token in range(len(NER)):
        if NER[token] in i:
            i = i.replace(NER[token], " tags" + str(token) + " ")

    raw = i.lower()
    tokens = tokenizer.tokenize(raw)

    for word in range(len(tokens)):
        for token in range(len(NER)):
            if tokens[word] == (" tags" + str(token) + " "):
                tokens[word] = NER[token]

    # remove stop words from tokens
    stopped_tokens = [i for i in tokens if not i in en_stop]

    # add tokens to list
    texts.append(stopped_tokens)

# turn our tokenized documents into a id <-> term dictionary
dictionary = corpora.Dictionary(texts)

# convert tokenized documents into a document-term matrix
corpus = [dictionary.doc2bow(text) for text in texts]

# generate LDA model
ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=5, id2word=dictionary, passes=20)
lda_raw = ldamodel.print_topics(num_topics=2, num_words=100)

try:
    # Preprocessing
    # Gathering each words in LDA

    nstr = re.sub(r'[+|""|,|\\|'']', r'', str(lda_raw[1]))
    tokens = word_tokenize(nstr)

    lda_tokens = []
    for i in tokens:
        lda_tokens.append(str(i).split('*'))

    lda_words = []
    index = 0
    for i in lda_tokens:
        if index >= 2 and index < len(lda_tokens) - 2:
            lda_words.append(i[1])
        index += 1

        # Part-of-Speech Tagging in Templating is divided into two.
        # The POS tagging from LDA to filter out the topic in the corpora and the Original Copy which is the template that
        # change the
except IndexError:
    print("Error Occur")

# from LDA
lda_tagging = pos_tag(lda_words)
print(lda_tagging)
"""


#!/usr/bin/env python3
# -*- encoding=utf-8 -*-

# description:
# author:jack
# create_time: 2018/1/3

"""
    desc:pass
"""
import os
import logging
import traceback
from dueros.Bot import Bot
from dueros.card.TextCard import TextCard
from dueros.card.ImageCard import ImageCard
from dueros.card.StandardCard import StandardCard
from dueros.samples.xichuangzhu.Poem import Poem
from dueros.directive.BaseDirective import BaseDirective
import json
import random

class Bot(Bot):
    def launchRequest(self):
        standardCard = self.get_one_poem_standardCard()
        return {
            'card': standardCard,
            'outputSpeech': '<speak> %s </speak>' % standardCard.data["content"]
        }

    def nextIntentRequest(self):
        standardCard = self.get_one_poem_standardCard()
        return {
            'card': standardCard,
            'outputSpeech': '<speak> %s </speak>' % standardCard.data["content"]
        }

    def previousIntentRequest(self):
        standardCard = self.get_one_poem_standardCard()
        return {
            'card': standardCard,
            'outputSpeech': '<speak> %s </speak>' % standardCard.data["content"]
        }

    def seemoreIntentRequest(self):
        title = ""
        try:
            anchorText = (self._request.data["context"]["Screen"]["card"]["anchorText"])
        except Exception as e:
            standardCard = self.get_one_poem_standardCard()
            return {
                'card': standardCard,
                'outputSpeech': '<speak> %s </speak>' % standardCard.data["content"]
            }
        base_directive = BaseDirective("Display.RenderSwanView")
        poem_see_more = ""
        try:
            poem_see_more = Poem.get_see_more(gushiwen_url=anchorText)
        except Exception as e:
            print (traceback.print_exc())
        if poem_see_more:
            logging.info('poem_see_more:' + json.dumps(poem_see_more))
            standardCard = self.get_one_poem_standardCard()
            standardCard.data["title"] = poem_see_more["chuzi"]
            standardCard.data["content"] = poem_see_more["text"] + "\n\n" + poem_see_more["yiwen"]
            standardCard.data["image"] = Poem.get_one_random_image()
            return {
                'card': standardCard,
                'outputSpeech': '<speak> %s </speak>' % (poem_see_more["text"])
            }
        else:
            return {
                'outputSpeech': '<speak> 抱歉， 未找到这首诗 </speak>'
            }

    def get_one_poem_standardCard(self):
        standardCard = StandardCard()
        one_poem_item = Poem.get_one_poem_item()
        standardCard.data["title"] = one_poem_item["chuzi"]
        standardCard.data["content"] = one_poem_item["mingju"]
        standardCard.data['anchorText'] = one_poem_item["url"]
        standardCard.data["image"] = Poem.get_one_random_image()
        return standardCard

    def __init__(self, data):
        super(Bot, self).__init__(data)

        self.add_launch_handler(self.launchRequest)

        self.add_intent_handler('ai.dueros.common.next_intent', self.nextIntentRequest)

        self.add_intent_handler('ai.dueros.common.previous_intent', self.previousIntentRequest)

        self.add_intent_handler('xichuangzhu----see_more_about_it', self.seemoreIntentRequest)

    pass

if __name__ == '__main__':
    pass


# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""
Common functionalities for AMT Driver
"""
import logging
import pywsman
import xmltodict
from ast import literal_eval
from xml.etree import ElementTree
from wry import data_structures
from wry import exceptions
from wry.decorators import retry, add_client_options
from wry.config import RESOURCE_URIs
from wry.data_structures import _strip_namespace_prefixes, WryDict

_SOAP_ENVELOPE = 'http://www.w3.org/2003/05/soap-envelope'

LOG = logging.getLogger(__name__)

AMT_PROTOCOL_PORT_MAP = {
    'http': 16992,
    'https': 16993,
}

def _validate(doc, silent=False):
    if doc is None:
        raise exceptions.AMTConnectFailure
    if not silent:
        if doc.is_fault():
            raise exceptions.WSManFault(doc)
    return doc

@add_client_options
@retry
def wsman_get(client, resource_uri, options=None, silent=False):
    '''Get target server info'''
    doc = client.get(options, resource_uri)
    return _validate(doc, silent=silent)

@add_client_options
@retry
def wsman_pull(client, resource_uri, options=None, wsman_filter=None, context=None, silent=False):
    '''Get target server info'''
    doc = client.pull(options, wsman_filter, resource_uri, context)
    return _validate(doc, silent=silent)

@add_client_options
@retry
def wsman_enumerate(client, resource_uri, options=None, wsman_filter=None, silent=False):
    '''Get target server info'''
    doc = client.enumerate(options, wsman_filter, resource_uri)
    return _validate(doc, silent=silent)

@add_client_options
@retry
def wsman_put(client, resource_uri, data, options=None, silent=False):
    '''Invoke method on target server
    :param silent: Ignore WSMan errors, and return the document anyway. Does not
    ignore the endpoint being down.
    '''
    doc = client.put(options, resource_uri, str(data), len(data))
    return _validate(doc, silent=silent)

@add_client_options
@retry
def wsman_invoke(client, resource_uri, method, data=None, options=None, silent=False):
    '''Invoke method on target server.'''
    doc = client.invoke(options, resource_uri, str(method), pywsman.create_doc_from_string(str(data)))
    return _validate(doc, silent=silent)

def get_resource(client, resource_name, options=None):
    '''
    '''
    uri = RESOURCE_URIs[resource_name]
    doc = wsman_get(client, uri, options=options)
    return WryDict(doc)

def enumerate_resource(client, resource_name, wsman_filter=None, options=None):
    '''
    class.
    '''
    uri = RESOURCE_URIs[resource_name]
    doc = wsman_enumerate(client, uri, options=options) # Add in relevant kwargs... filter?
    doc = WryDict(doc)
    context = doc['EnumerateResponse']['EnumerationContext']
    ended = False
    output = {resource_name: []}
    while ended is False:
        doc = wsman_pull(client, uri, context=str(context), options=options)
        response = WryDict(doc)['PullResponse']
        ended = response.pop('EndOfSequence', False)
        output[resource_name].append(response['Items'][resource_name])
    return output

def put_resource(client, indict, options=None, uri=None, silent=False):
    '''
    Given a dict or  describing a wsman resource, post this resource to the client.
    :returns: data_structures.WryDict
    :param indict: A dictionary or dictionary-like object (eg.
    common.RESOURCE_URIs.
    :param uri: If a mapping does not exist in common.RESOURCE_URIs, the resource URI can be specified manually here.
    :param mappings: A dictionary providing extra mappings between resource names and URIs.
    '''
    if not uri:
        uri = RESOURCE_URIs[indict.keys()[0]] # Possible to support multiple simply here?
    data = indict.as_xml()
    doc = wsman_put(client, uri, data, options=options, silent=silent)
    return WryDict(doc)

def invoke_method(client, method, input_dict, options=None):
    resource_name = input_dict.keys()[0]
    input_dict = WryDict(input_dict).with_namespaces()
    data = input_dict[resource_name]
    uri = data.pop(u'@xmlns')
    data.values()[0][u'@xmlns'] = uri
    xml = xmltodict.unparse(data, full_document=False, pretty=True)
    doc = wsman_invoke(client, RESOURCE_URIs[resource_name], method, xml, options=options)
    returned = WryDict(doc)
    return_value = returned[returned.keys()[0]]['ReturnValue']
    if return_value != 0:
        raise exceptions.NonZeroReturn(return_value)
    return returned


import logging
import uvicorn
import pathlib

class logger_class:
    def __init__(self):
        self.logfile = "test.log"
        self.logger = logging.getLogger("uvicorn.access")
        self.setup_loging()

    def setup_loging(self):
        logger = logging.getLogger("uvicorn.access")
        console_formatter = uvicorn.logging.ColourizedFormatter(
            "{asctime} {levelprefix} : {message}",
            style="{", use_colors=True)
        logger.handlers[0].setFormatter(console_formatter)
        logfile = pathlib.Path(self.logfile)
        logfile.touch()
        logfile.chmod(0o666)
        handler = logging.FileHandler(filename=self.logfile)
        handler.setFormatter(logging.Formatter(
            "%(asctime)s %(levelname)8s : %(message)s"))
        handler.setLevel(logging.WARN)
        logger.addHandler(handler)

        logger = logging.getLogger("uvicorn")
        console_formatter = uvicorn.logging.ColourizedFormatter(
            "{asctime} {levelprefix} : {message}",
            style="{", use_colors=True)
        logger.handlers[0].setFormatter(console_formatter)

    def error(self, message):
        self.logger.error(message)

    def info(self, message):
        self.logger.info(message)

    def debug(self, message):
        self.logger.debug(message)

    def warning(self, message):
        self.logger.warning(message)

logger = logger_class()


# -*- coding:UTF-8 -*-
#*********************************************
#------->>>>>>Author:秋某人的傻逼                *
#------->>>>>>Name:熊猫最爱皮卡丘                 *
#------->>>>>>Target:端口实时监控工具        *
#*********************************************

#--------------------/工具编写思路/--------------------
#1.开始并发
#--------------------/工具编写思路/--------------------

import os
import glob
import datetime
import subprocess

print('---------------------------------------------')
print('欢迎使用端口实时监控工具，本工具版本号：V3.0')
print('---------------------------------------------')
# ip伪线程设置
print('本软件默认开启所有IP并发扫描！请控制IP数量或者升级服务器配置以免宕机')
print('支持线程是：1、3、5、15、17、51、85、255、257、771、1285、3855、4369、13107、21845，请不要输入错误！')
thread = input('请输入端口扫描线程：')
thread = int(thread)
# 可以被整除：1、3、5、15、17、51、85、255、257、771、1285、3855、4369、13107、21845。
# ------------------------
# ip伪并发，65535切分模块
# ------------------------
def ip_ct(thread):
    print('INFO: The Thread is ' + str(thread))
    Number = int(65535 / thread)
    print(Number)
    i = 1
    dirpath = 'ports/'
    # 判断路径是否存在，然后判断文件是否存在，如果存在就将文件删除，如果路径不存在就创建目录
    if os.path.exists('ports'):
        if os.listdir('ports'):
            print('文件夹Ports已经存在，正在对文件进行清空...')
            for file in os.walk('ports'):
                print('os.walk返回对象' + str(file))
                for items in file[2]:
                    print('目标文件：' + items)
                    os.remove(dirpath + items)
                    print(items + '| 文件已删除')
        else:
            print('Ports目录已存在，且无文件！')
    else:
        os.mkdir('ports')
        print('Ports目录生成成功！')
    # 根据输入的port线程数目，然后对65535进行分割，然后生成文件，对文件进行列目录
    while i <= thread:
        count_original = Number * (i - 1)
        count = Number * i
        print('当前是生成第' + str(i) + '个线程')
        for port in range(count_original, count):
            with open('ports/port_' + str(i) + '.txt', 'a+') as f:
                f.write(str(port))
                f.write('\n')
                f.close()
        i += 1
    for filename in glob.glob(r'ports/*.txt'):
        print('文件：' + filename + '| 生成成功！')
    print('IP线程预分类模块完成！')
    #创建目录模块
def mkdir():
    now = datetime.datetime.now()
    path = 'portscan_results/' + str(now.year) + '-' + str(now.month) + '-' + str(now.day)
    # 去除首位空格
    # path = path.strip()
    # 去除尾部 \ 符号
    # path = path.rstrip("\\")
    isExists = os.path.exists(path)
    # 判断结果
    if not isExists:
        os.makedirs(path)
        print(path + ' 创建成功')
        return True
    else:
        print(path + ' 目录已存在')
        return False
ip_ct(thread)
mkdir()

for ip in open('hosts.txt','r'):
    listfile = os.listdir('ports')
    len_listfile = len(listfile)
    for x in range(0,len_listfile):
        for port in open('ports/'+ listfile[x],'r'):
            subprocess.Popen("python ./Real_Time_Port_Scaner.py " + ip + port)


from websocket_server import WebsocketServer, WebsocketClientBehavior

# Called for every client connecting (after handshake)
def new_client(client, server):
  print("New client connected and was given id %d" % client.id)
  server.send_message_to_all("Hey all, a new client has joined us")

# Called for every client disconnecting
def client_left(client, server):
	print("Client(%d) disconnected" % client.id)

# Called when a client sends a message but overriden)
def message_received(client, server, message):
	if len(message) > 200:
		message = message[:200]+'..'
	print("Client(%d) from %s said: %s" % (client.id, client.origin, message))

class hdrBhv(WebsocketClientBehavior):
  '''
    Set a different behavior
  '''
  def on_text(self, msg):
    '''
      if the text sent through msg is in the request headers
      respond with that header.
    '''
    if msg in self.handler.headers:
      self.send_text(self.handler.headers[msg]);
    else:
      print('Client(%d) from %s said: %s' % (self.id, self.origin, msg));
  # on_text and on_open keeps with the same behavior

class revBhv(WebsocketClientBehavior):
  '''
    Set a different behavior
  '''
  def on_text(self, msg):
    '''
      if the text sent through msg is in the request headers
      respond with that header.
    '''
    self.send_text('.'.join(msg[::-1]));
  # on_text and on_open keeps with the same behavior

PORT=9001
server = WebsocketServer(PORT)
# override the default behavior for clients connection to ws://host:PORT/hdr
server.behaviors['/headers'] = hdrBhv;
server.behaviors['/reverse'] = revBhv;

# default handlers, will be called if the connect to the server with a different address
server.set_fn_new_client(new_client)
server.set_fn_client_left(client_left)
server.set_fn_message_received(message_received)
server.run_forever()


import tensorflow as tf
import input_data
import numpy as np
from PIL import Image

batch_size = 64
z_dim = 100
learning_rate = 0.001
beta1 = 0.5
epochs = 5000

def model_inputs(image_width, image_height, image_channels, z_dim):
    # Real imag
    inputs_real = tf.placeholder(tf.float32, (None, image_width, image_height, image_channels), name='input_real')

    # input z

    inputs_z = tf.placeholder(tf.float32, (None, z_dim), name='input_z')

    # Learning rate
    learning_rate = tf.placeholder(tf.float32, name='lr')

    return inputs_real, inputs_z, learning_rate

def generator(input_z, out_channel_dim, is_train=True):
    with tf.variable_scope('generator', reuse=not is_train):
        x0 = tf.layers.dense(input_z, 4 * 4 * 512)
        x0 = tf.reshape(x0, (-1, 4, 4, 512))
        bn0 = tf.layers.batch_normalization(x0, training=is_train)
        relu0 = tf.nn.relu(bn0)

        # 反卷积
        x1 = tf.layers.conv2d_transpose(relu0, 256, 4, strides=1, padding='valid')
        bn1 = tf.layers.batch_normalization(x1, training=is_train)
        relu1 = tf.nn.relu(bn1)

        x2 = tf.layers.conv2d_transpose(relu1, 512, 3, strides=2, padding='same')
        bn2 = tf.layers.batch_normalization(x2, training=is_train)
        relu2 = tf.nn.relu(bn2)

        logits = tf.layers.conv2d_transpose(relu2, out_channel_dim, 3, strides=2, padding='same')
        out = tf.tanh(logits)

    return out

def discriminator(images, reuse=False):
    """
    Create the discriminator network
    :param images: Tensor of input image(s)
    :param reuse: Boolean if the weights should be reused
    :return: Tuple of (tensor output of the discriminator, tensor logits of the discriminator)
    """
    # TODO: Implement Function

    # scope here

    with tf.variable_scope('discriminator', reuse=reuse):
        alpha = 0.2  # leak relu coeff

        # drop out probability
        keep_prob = 0.8

        # input layer 28 * 28 * color channel
        x1 = tf.layers.conv2d(images, 128, 5, strides=2, padding='same',
                              kernel_initializer=tf.contrib.layers.xavier_initializer(seed=2))
        # No batch norm here
        # leak relu here / alpha = 0.2
        relu1 = tf.maximum(alpha * x1, x1)
        # applied drop out here
        drop1 = tf.nn.dropout(relu1, keep_prob=keep_prob)
        # 14 * 14 * 128

        # Layer 2
        x2 = tf.layers.conv2d(drop1, 256, 5, strides=2, padding='same',
                              kernel_initializer=tf.contrib.layers.xavier_initializer(seed=2))
        # employ batch norm here
        bn2 = tf.layers.batch_normalization(x2, training=True)
        # leak relu
        relu2 = tf.maximum(alpha * bn2, bn2)
        drop2 = tf.nn.dropout(relu2, keep_prob=keep_prob)

        # 7 * 7 * 256

        # Layer3
        x3 = tf.layers.conv2d(drop2, 512, 5, strides=2, padding='same',
                              kernel_initializer=tf.contrib.layers.xavier_initializer(seed=2))
        bn3 = tf.layers.batch_normalization(x3, training=True)
        relu3 = tf.maximum(alpha * bn3, bn3)
        drop3 = tf.nn.dropout(relu3, keep_prob=keep_prob)
        # 4 * 4 * 512

        # Output
        # Flatten
        flatten = tf.reshape(relu3, (-1, 4 * 4 * 512))
        logits = tf.layers.dense(flatten, 1)
        # activation
        out = tf.nn.sigmoid(logits)

    return out, logits

def model_loss(input_real, input_z, out_channel_dim):
    """
    Get the loss for the discriminator and generator
    :param input_real: Images from the real dataset
    :param input_z: Z input
    :param out_channel_dim: The number of channels in the output image
    :return: A tuple of (discriminator loss, generator loss)
    """
    # TODO: Implement Function

    g_model = generator(input_z, out_channel_dim, is_train=True)

    g_model1 = generator(input_z, out_channel_dim, is_train=False)

    d_model_real, d_logits_real = discriminator(input_real, reuse=False)

    d_model_fake, d_logits_fake = discriminator(g_model, reuse=True)

    ## add smooth here

    smooth = 0.1
    d_loss_real = tf.reduce_mean(
        tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_real,
                                                labels=tf.ones_like(d_model_real) * (1 - smooth)))

    d_loss_fake = tf.reduce_mean(
        tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_fake, labels=tf.zeros_like(d_model_fake)))

    g_loss = tf.reduce_mean(
        tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logits_fake,
                                                labels=tf.ones_like(d_model_fake)))

    d_loss = d_loss_real + d_loss_fake

    return d_loss, g_loss, g_model1

def model_opt(d_loss, g_loss, learning_rate, beta1):
    """
    Get optimization operations
    :param d_loss: Discriminator loss Tensor
    :param g_loss: Generator loss Tensor
    :param learning_rate: Learning Rate Placeholder
    :param beta1: The exponential decay rate for the 1st moment in the optimizer
    :return: A tuple of (discriminator training operation, generator training operation)
    """

    t_vars = tf.trainable_variables()
    d_vars = [var for var in t_vars if var.name.startswith('discriminator')]
    g_vars = [var for var in t_vars if var.name.startswith('generator')]

    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

    with tf.control_dependencies(update_ops):
        d_train_opt = tf.train.AdamOptimizer(learning_rate, beta1=beta1).minimize(d_loss, var_list=d_vars)
        g_train_opt = tf.train.AdamOptimizer(learning_rate, beta1=beta1).minimize(g_loss, var_list=g_vars)

    return d_train_opt, g_train_opt

def train(epoch_count, batch_size, z_dim, learning_rate, beta1, data_shape):
    input_real, input_z, lr = model_inputs(data_shape[1], data_shape[2], data_shape[3], z_dim)
    d_loss, g_loss, g_out = model_loss(input_real, input_z, data_shape[-1])
    d_opt, g_opt = model_opt(d_loss, g_loss, learning_rate, beta1)
    steps = 0
    losses = []
    mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for epoch_i in range(epoch_count):
            steps += 1
            batch = mnist.train.next_batch(batch_size)
            labels = batch[1]
            batch = batch[0]
            train_batch = []
            for index, label in enumerate(labels):
                if label[2] == 1:
                    train_batch.append(batch[index])
            input_batch_size = len(train_batch)
            train_batch = np.array(train_batch)
            train_batch = np.reshape(train_batch, (-1, data_shape[1], data_shape[2], data_shape[3]))
            batch_z = np.random.uniform(-1, 1, size=(input_batch_size, z_dim))
            _ = sess.run(d_opt, feed_dict={input_real: train_batch, input_z: batch_z, lr: learning_rate})
            _ = sess.run(g_opt, feed_dict={input_real: train_batch, input_z: batch_z, lr: learning_rate})
            if steps % 10 == 0:
                train_loss_d = d_loss.eval({input_real: train_batch, input_z: batch_z})
                train_loss_g = g_loss.eval({input_real: train_batch, input_z: batch_z})

                losses.append((train_loss_d, train_loss_g))

                print("Epoch {}/{}...".format(epoch_i + 1, epochs),
                      "Discriminator Loss: {:.4f}...".format(train_loss_d),
                      "Generator Loss: {:.4f}".format(train_loss_g))
            if steps % 100 == 0:
                img = g_out.eval({input_z: batch_z})
                img = img[-1] * 128 + 128
                img = img.astype(int)
                img = img[:, :, 0]
                im = Image.fromarray(img).convert('L')
                im.save("result_{}.png".format(steps))
                # im.show()

if __name__ == "__main__":
    train(epochs, batch_size, z_dim, learning_rate, beta1, [0, 28, 28, 1])


#
# Copyright (c) 2013, Prometheus Research, LLC
# Released under MIT license, see `LICENSE` for details.
#

from setuptools import setup, find_packages

NAME = "Cogs"
VERSION = "0.4.4"
DESCRIPTION = """Toolkit for developing command-line utilities in Python"""
LONG_DESCRIPTION = open('README', 'r').read()
AUTHOR = """Kirill Simonov (Prometheus Research, LLC)"""
AUTHOR_EMAIL = "xi@resolvent.net"
LICENSE = "MIT"
URL = "https://github.com/prometheusresearch/cogs"
DOWNLOAD_URL = "http://pypi.python.org/pypi/Cogs"
CLASSIFIERS = [
    "Development Status :: 3 - Alpha",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "Intended Audience :: System Administrators",
    "License :: OSI Approved :: MIT License",
    "Operating System :: POSIX :: Linux",
    "Programming Language :: Python",
    "Programming Language :: Python :: 2",
    "Programming Language :: Python :: 3",
    "Topic :: Utilities",
]
PACKAGES = find_packages('src')
PACKAGE_DIR = {'': 'src'}
NAMESPACE_PACKAGES = ['cogs']
INSTALL_REQUIRES = ['setuptools', 'PyYAML']
ENTRY_POINTS = {
    'console_scripts': [
        'cogs = cogs.run:main',
    ],
    'cogs.extensions': [],
}
USE_2TO3 = True

setup(name=NAME,
      version=VERSION,
      description=DESCRIPTION,
      author=AUTHOR,
      author_email=AUTHOR_EMAIL,
      license=LICENSE,
      url=URL,
      download_url=DOWNLOAD_URL,
      classifiers=CLASSIFIERS,
      packages=PACKAGES,
      package_dir=PACKAGE_DIR,
      namespace_packages=NAMESPACE_PACKAGES,
      install_requires=INSTALL_REQUIRES,
      entry_points=ENTRY_POINTS,
      use_2to3=USE_2TO3)


import json

from django.db import models
from django.core.exceptions import ValidationError
from django.contrib.contenttypes.fields import GenericRelation

from mezzanine.pages.page_processors import processor_for

from hs_core.models import BaseResource, ResourceManager
from hs_core.models import resource_processor, CoreMetaData, AbstractMetaDataElement
from hs_core.hydroshare.utils import get_resource_file_name_and_extension

# Define original spatial coverage metadata info
class OriginalCoverage(AbstractMetaDataElement):
    PRO_STR_TYPES = (
        ('', '---------'),
        ('WKT String', 'WKT String'),
        ('Proj4 String', 'Proj4 String')
    )

    term = 'OriginalCoverage'
    """
    _value field stores a json string. The content of the json as box coverage info
         _value = "{'northlimit':northenmost coordinate value,
                    'eastlimit':easternmost coordinate value,
                    'southlimit':southernmost coordinate value,
                    'westlimit':westernmost coordinate value,
                    'units:units applying to 4 limits (north, east, south & east),
                    'projection': name of the projection (optional)}"
    """
    _value = models.CharField(max_length=1024, null=True)
    projection_string_type = models.CharField(max_length=20, choices=PRO_STR_TYPES, null=True)
    projection_string_text = models.TextField(null=True, blank=True)
    datum = models.CharField(max_length=300, blank=True)

    class Meta:
        # OriginalCoverage element is not repeatable
        unique_together = ("content_type", "object_id")

    @property
    def value(self):
        return json.loads(self._value)

    @classmethod
    def create(cls, **kwargs):
        """
        The '_value' subelement needs special processing. (Check if the 'value' includes the
        required information and convert 'value' dict as Json string to be the '_value'
        subelement value.) The base class create() can't do it.

        :param kwargs: the 'value' in kwargs should be a dictionary
                       the '_value' in kwargs is a serialized json string
        """
        value_arg_dict = None
        if 'value' in kwargs:
            value_arg_dict = kwargs['value']
        elif '_value' in kwargs:
            value_arg_dict = json.loads(kwargs['_value'])

        if value_arg_dict:
            # check that all the required sub-elements exist and create new original coverage meta
            for value_item in ['units', 'northlimit', 'eastlimit', 'southlimit', 'westlimit']:
                if value_item not in value_arg_dict:
                    raise ValidationError("For original coverage meta, one or more bounding "
                                          "box limits or 'units' is missing.")

            value_dict = {k: v for k, v in value_arg_dict.iteritems()
                          if k in ('units', 'northlimit', 'eastlimit', 'southlimit',
                                   'westlimit', 'projection')}

            value_json = json.dumps(value_dict)
            if 'value' in kwargs:
                del kwargs['value']
            kwargs['_value'] = value_json
            return super(OriginalCoverage, cls).create(**kwargs)
        else:
            raise ValidationError('Coverage value is missing.')

    @classmethod
    def update(cls, element_id, **kwargs):
        """
        The '_value' subelement needs special processing. (Convert 'value' dict as Json string
        to be the '_value' subelement value) and the base class update() can't do it.

        :param kwargs: the 'value' in kwargs should be a dictionary
        """

        ori_cov = OriginalCoverage.objects.get(id=element_id)
        if 'value' in kwargs:
            value_dict = ori_cov.value

            for item_name in ('units', 'northlimit', 'eastlimit', 'southlimit',
                              'westlimit', 'projection'):
                if item_name in kwargs['value']:
                    value_dict[item_name] = kwargs['value'][item_name]

            value_json = json.dumps(value_dict)
            del kwargs['value']
            kwargs['_value'] = value_json
            super(OriginalCoverage, cls).update(element_id, **kwargs)

# Define netCDF variable metadata
class Variable(AbstractMetaDataElement):
    # variable types are defined in OGC enhanced_data_model_extension_standard
    # left is the given value stored in database right is the value for the drop down list
    VARIABLE_TYPES = (
        ('Char', 'Char'),  # 8-bit byte that contains uninterpreted character data
        ('Byte', 'Byte'),  # integer(8bit)
        ('Short', 'Short'),  # signed integer (16bit)
        ('Int', 'Int'),  # signed integer (32bit)
        ('Float', 'Float'),  # floating point (32bit)
        ('Double', 'Double'),  # floating point(64bit)
        ('Int64', 'Int64'),  # integer(64bit)
        ('Unsigned Byte', 'Unsigned Byte'),
        ('Unsigned Short', 'Unsigned Short'),
        ('Unsigned Int', 'Unsigned Int'),
        ('Unsigned Int64', 'Unsigned Int64'),
        ('String', 'String'),  # variable length character string
        ('User Defined Type', 'User Defined Type'),  # compound, vlen, opaque, enum
        ('Unknown', 'Unknown')
    )
    term = 'Variable'
    # required variable attributes
    name = models.CharField(max_length=1000)
    unit = models.CharField(max_length=1000)
    type = models.CharField(max_length=1000, choices=VARIABLE_TYPES)
    shape = models.CharField(max_length=1000)
    # optional variable attributes
    descriptive_name = models.CharField(max_length=1000, null=True, blank=True,
                                        verbose_name='long name')
    method = models.TextField(null=True, blank=True, verbose_name='comment')
    missing_value = models.CharField(max_length=1000, null=True, blank=True)

    def __unicode__(self):
        return self.name

    @classmethod
    def remove(cls, element_id):
        raise ValidationError("The variable of the resource can't be deleted.")

# Define the netCDF resource
class NetcdfResource(BaseResource):
    objects = ResourceManager("NetcdfResource")

    @property
    def metadata(self):
        md = NetcdfMetaData()
        return self._get_metadata(md)

    @classmethod
    def get_supported_upload_file_types(cls):
        # 3 file types are supported
        return (".nc",)

    @classmethod
    def allow_multiple_file_upload(cls):
        # can upload only 1 file
        return False

    @classmethod
    def can_have_multiple_files(cls):
        # can have only 1 file
        return False

    # add resource-specific HS terms
    def get_hs_term_dict(self):
        # get existing hs_term_dict from base class
        hs_term_dict = super(NetcdfResource, self).get_hs_term_dict()
        # add new terms for NetCDF res
        hs_term_dict["HS_NETCDF_FILE_NAME"] = ""
        for res_file in self.files.all():
            _, f_fullname, f_ext = get_resource_file_name_and_extension(res_file)
            if f_ext.lower() == '.nc':
                hs_term_dict["HS_NETCDF_FILE_NAME"] = f_fullname
                break
        return hs_term_dict

    class Meta:
        verbose_name = 'Multidimensional (NetCDF)'
        proxy = True

processor_for(NetcdfResource)(resource_processor)

# define the netcdf metadata
class NetcdfMetaData(CoreMetaData):
    variables = GenericRelation(Variable)
    ori_coverage = GenericRelation(OriginalCoverage)

    @classmethod
    def get_supported_element_names(cls):
        # get the names of all core metadata elements
        elements = super(NetcdfMetaData, cls).get_supported_element_names()
        # add the name of any additional element to the list
        elements.append('Variable')
        elements.append('OriginalCoverage')
        return elements

    @property
    def resource(self):
        return NetcdfResource.objects.filter(object_id=self.id).first()

    def has_all_required_elements(self):
        if not super(NetcdfMetaData, self).has_all_required_elements():  # check required meta
            return False
        if not self.variables.all():
            return False
        if not (self.coverages.all().filter(type='box').first() or
                self.coverages.all().filter(type='point').first()):
            return False
        return True

    def get_required_missing_elements(self):  # show missing required meta
        missing_required_elements = super(NetcdfMetaData, self).get_required_missing_elements()
        if not (self.coverages.all().filter(type='box').first() or
                self.coverages.all().filter(type='point').first()):
            missing_required_elements.append('Spatial Coverage')
        if not self.variables.all().first():
            missing_required_elements.append('Variable')

        return missing_required_elements

    def get_xml(self, pretty_print=True):
        from lxml import etree
        # get the xml string representation of the core metadata elements
        xml_string = super(NetcdfMetaData, self).get_xml(pretty_print=pretty_print)

        # create an etree xml object
        RDF_ROOT = etree.fromstring(xml_string)

        # get root 'Description' element that contains all other elements
        container = RDF_ROOT.find('rdf:Description', namespaces=self.NAMESPACES)

        # inject netcdf resource specific metadata element 'variable' to container element
        for variable in self.variables.all():
            md_fields = {
                "md_element": "netcdfVariable",
                "name": "name",
                "unit": "unit",
                "type": "type",
                "shape": "shape",
                "descriptive_name": "longName",
                "method": "comment",
                "missing_value": "missingValue"
            }  # element name : name in xml
            self.add_metadata_element_to_xml(container, variable, md_fields)

        if self.ori_coverage.all().first():
            ori_cov_obj = self.ori_coverage.all().first()
            hsterms_ori_cov = etree.SubElement(container, '{%s}spatialReference' %
                                               self.NAMESPACES['hsterms'])
            cov_term = '{%s}' + 'box'
            hsterms_coverage_terms = etree.SubElement(hsterms_ori_cov, cov_term %
                                                      self.NAMESPACES['hsterms'])

            hsterms_ori_cov_rdf_Description = etree.SubElement(hsterms_coverage_terms, '{%s}value' %
                                                               self.NAMESPACES['rdf'])
            cov_box = ''

            # add extent info
            if ori_cov_obj.value:
                cov_box = 'northlimit=%s; eastlimit=%s; southlimit=%s; westlimit=%s; unit=%s' \
                        % (ori_cov_obj.value['northlimit'], ori_cov_obj.value['eastlimit'],
                           ori_cov_obj.value['southlimit'], ori_cov_obj.value['westlimit'],
                           ori_cov_obj.value['units'])

            if ori_cov_obj.value.get('projection'):
                cov_box += '; projection_name={}'.format(ori_cov_obj.value['projection'])

            if ori_cov_obj.projection_string_text:
                cov_box += '; projection_string={}'.format(ori_cov_obj.projection_string_text)

            if ori_cov_obj.datum:
                cov_box += '; datum={}'.format(ori_cov_obj.datum)

            hsterms_ori_cov_rdf_Description.text = cov_box

        return etree.tostring(RDF_ROOT, pretty_print=pretty_print)

    def add_metadata_element_to_xml(self, root, md_element, md_fields):
        from lxml import etree
        element_name = md_fields.get('md_element') if md_fields.get('md_element') \
            else md_element.term

        hsterms_newElem = etree.SubElement(
            root,
            "{{{ns}}}{new_element}".format(ns=self.NAMESPACES['hsterms'], new_element=element_name))

        hsterms_newElem_rdf_Desc = etree.SubElement(
            hsterms_newElem, "{{{ns}}}Description".format(ns=self.NAMESPACES['rdf']))

        for md_field in md_fields.keys():
            if hasattr(md_element, md_field):
                attr = getattr(md_element, md_field)
                if attr:
                    field = etree.SubElement(hsterms_newElem_rdf_Desc,
                                             "{{{ns}}}{field}".format(ns=self.NAMESPACES['hsterms'],
                                                                      field=md_fields[md_field]))
                    field.text = str(attr)

    def delete_all_elements(self):
        super(NetcdfMetaData, self).delete_all_elements()
        self.ori_coverage.all().delete()
        self.variables.all().delete()


## READ LOW RESOLUTION MITGCM RUN

import numpy as np
import matplotlib.pyplot as plt
from netCDF4 import Dataset as ncread
import glob

path = '/home/jupiter/ocean/fenwick/baroclinic_gyre/baroclinic_128x128x4_uvTuvT_mk4/run_370/mnc_test_00'
#loop over 13,14,15,16

files = []

for folder in [13,14,15,16]:
    files.extend(glob.glob(path+str(folder)+'/state*.nc'))

x = np.zeros((len(files),33))
y = np.zeros((len(files),33))

#get dimensions

for i,fil in zip(range(len(files)),files):
    dat = ncread(fil)
    x[i,:] = dat.variables['X'][:]
    y[i,:] = dat.variables['Y'][:]

#read time only once
time = dat.variables['T'][:]
dat.close()
print('Dimensions read.')

x = np.sort(np.array(list(set([i for i in x.flatten()]))))
y = np.sort(np.array(list(set([i for i in y.flatten()]))))

## preallocate surface temperature
# use only 3000 last time steps (due to spin up)
tlength = 6000
temp = np.zeros((tlength,y.shape[0],x.shape[0]))

for i,fil in zip(range(len(files)),files):
    print('File no. '+str(i+1)+' of '+str(len(files)))
    dat = ncread(fil)
    datx = dat.variables['X'][:]
    daty = dat.variables['Y'][:]

    ys = np.where(y == daty[0])[0][0]
    xs = np.where(x == datx[0])[0][0]

    temp[:,ys:ys+daty.shape[0],xs:xs+datx.shape[0]]\
    = dat.variables['Temp'][:][-tlength:,0,:,:]

## get rid of boundaries
#adapt dimensions
# use only every other point in time to get daily resolution

bx = 128
by = 128

temp = temp[::2,:by,:bx]
time = time[-tlength::2]
time = (time - time[0])/24./3600.

y = y[:by]
x = x[:bx]

## SAVE

np.save('python/gyres/temp_optlowres_sfc.npy',temp)
#np.save('python/gyres/temp_lowres_dim.npy',(time,y,x))
print('Files written.')


#!/usr/bin/env python
# coding: utf-8

# In[5]:

# Validation of assertions

import pandas as pd
import numpy as np
import math

df = pd.read_csv ('Downloads/oregon-crash2019.csv')

# In[53]:

# 41 to 46 Latitude Degrees

fasle_records=0
nan_records=0
for data in df['Latitude Degrees']:
    if (data<41 or data>46):
        fasle_records+=1
    elif math.isnan(data):
        nan_records+=1

print('fasle_records',fasle_records)
print('nan_records',nan_records)

# In[7]:

# 0 to 59 Latitude Minutes

fasle_records=0
nan_records=0
for data in df['Latitude Minutes']:
    if (data>=0 and data<=59):
        pass
    elif math.isnan(data):
        nan_records+=1
    else:
        fasle_records+=1

print('fasle_records',fasle_records)
print('nan_records',nan_records)

# In[8]:

# 0.00 to 59.99 Latitude Seconds

fasle_records=0
nan_records=0

for data in df['Latitude Seconds']:
    if (data>=0 and data<=59.99):
        pass
    elif math.isnan(data):
        nan_records+=1
    else:
        fasle_records+=1

print('false_records',fasle_records)
print('nan_records',nan_records)

# In[9]:

# A crash hour should be between 0-24

fasle_records=0
nan_records=0

for data in df['Crash Hour']:
    if (data>=0 and data<=24):
        pass
    elif math.isnan(data):
        nan_records+=1
    else:
        fasle_records+=1

print('false_records',fasle_records)
print('nan_records',nan_records)

# In[27]:

#On average, more crashes occur when the weather is clear.
from collections import defaultdict
dic=defaultdict(int)
weather = {
0: 'Unknown',
1: 'Clear',
2: 'Cloudy',
3: "Rain",
4: 'Sleet / Freezing Rain / Hail',
5: 'Fog',
6: 'Snow',
7: 'Dust',
8: 'Smoke',
9: 'Ash'
}

for data in df['Weather Condition']:
    if math.isnan(data):
        dic['nan_records']+=1
    else:
        dic[weather[data]]+=1

print(dic)

# In[46]:

from collections import defaultdict
count=defaultdict(int)
df1=list(zip(df['Weather Condition'], df['Road Surface Condition']))
road={
0:'Unknown',
1: 'Dry',
2: 'Wet',
3: 'Snow',
4: 'Ice'
}

wrong_entries=0
for data in df1:
    if data[0]==1 and data[1] in road:
        count[road[data[1]]]+=1
    elif math.isnan(data[1]):
        pass
    elif data[1] not in road:
        wrong_entries+=1

print('wrong entries', wrong_entries)
## Suprisingly more crashes occured when the when the weather condition is clear and road surface is dry (o_O)
print(str(dic['Clear'])+' crashes on clear weather condition with road surface condition: '+str([(i,dic[i]) for i in dic]))

# In[38]:

#All the crashes occurred during the day light.
from collections import defaultdict
light=defaultdict(int)
light_condition = {0: 'Unknown',
1: 'Daylight',
2: 'Darkness',
3: 'Darkness',
4: 'Dawn (Twilight)',
5: 'Dusk (Twilight)'}

for data in df['Light Condition']:
    if math.isnan(data):
        light['nan_records']+=1
    else:
        light[light_condition[data]]+=1

print(light)

# In[52]:

# Vehicle ID should be 7 digits

false_entries=0
correct_entries=0
nan_entries=0

for data in df['Vehicle ID']:
    if math.isnan(data):
        nan_entries+=1
    else:
        if len(str(int(data)))==7:
            correct_entries+=1
        else:
            false_entries+=1

print('false_entries',false_entries)
print('correct_entries',correct_entries)
print('nan_entries',nan_entries)


from django.urls import path
from . import views
from rest_framework_simplejwt import views as jwt_views

urlpatterns = [
    path('signup/', views.UserCreate.as_view(), name='account-create'),
    path('login/', views.LoginView.as_view(), name='login'),
    path('token/refresh/', jwt_views.TokenRefreshView.as_view(), name='token-refresh'),

    path('post_create/', views.PostCreateApiView.as_view(), name='post-create'),
    path('post/<int:pk>/like/', views.PostLikesUpdateApiView.as_view(), name='post-like'),
    path('post/<int:pk>/dislike/', views.PostDisLikesUpdateApiView.as_view(), name='post-dislike'),

    path('analytics/', views.AnalyticsAPIView.as_view(), name='analytics'),
    path('user_analytics/', views.UserAnalyticsAPIView.as_view(), name='user-analytics')
]


"""Integration for building changes on CircleCI."""

from __future__ import unicode_literals

import json
import logging

from django.contrib.auth.models import User
from django.db import IntegrityError, transaction
from django.utils.functional import cached_property
from django.utils.http import urlquote_plus
from django.utils.six.moves.urllib.request import urlopen
from djblets.avatars.services import URLAvatarService
from djblets.siteconfig.models import SiteConfiguration
from reviewboard.admin.server import get_server_url
from reviewboard.avatars import avatar_services
from reviewboard.extensions.hooks import SignalHook
from reviewboard.integrations import Integration
from reviewboard.reviews.models.status_update import StatusUpdate
from reviewboard.reviews.signals import review_request_published
from reviewboard.webapi.models import WebAPIToken

from rbintegrations.circleci.forms import CircleCIIntegrationConfigForm
from rbintegrations.util.urlrequest import URLRequest

logger = logging.getLogger(__name__)

class CircleCIIntegration(Integration):
    """Integrates Review Board with CircleCI."""

    name = 'CircleCI'
    description = 'Builds diffs posted to Review Board using CircleCI.'
    config_form_cls = CircleCIIntegrationConfigForm

    def initialize(self):
        """Initialize the integration hooks."""
        SignalHook(self, review_request_published,
                   self._on_review_request_published)

    @cached_property
    def icon_static_urls(self):
        """Return the icons used for the integration.

        Returns:
            dict:
            The icons for CircleCI.
        """
        from rbintegrations.extension import RBIntegrationsExtension

        extension = RBIntegrationsExtension.instance

        return {
            '1x': extension.get_static_url('images/circleci/icon.png'),
            '2x': extension.get_static_url('images/circleci/icon@2x.png'),
        }

    def _on_review_request_published(self, sender, review_request,
                                     changedesc=None, **kwargs):
        """Handle when a review request is published.

        Args:
            sender (object):
                The sender of the signal.

            review_request (reviewboard.reviews.models.review_request.
                            ReviewRequest):
                The review request which was published.

            changedesc (reviewboard.changedescs.models.ChangeDescription,
                        optional):
                The change description associated with this publish.

            **kwargs (dict):
                Additional keyword arguments.
        """
        # Only build changes against GitHub or Bitbucket repositories.
        repository = review_request.repository

        if not repository or not repository.hosting_account:
            return

        service_name = repository.hosting_account.service_name

        if service_name not in ('github', 'bitbucket'):
            return

        diffset = review_request.get_latest_diffset()

        # Don't build any review requests that don't include diffs.
        if not diffset:
            return

        # If this was an update to a review request, make sure that there was a
        # diff update in it.
        if changedesc is not None:
            fields_changed = changedesc.fields_changed

            if ('diff' not in fields_changed or
                'added' not in fields_changed['diff']):
                return

        matching_configs = [
            config
            for config in self.get_configs(review_request.local_site)
            if config.match_conditions(form_cls=self.config_form_cls,
                                       review_request=review_request)
        ]

        if not matching_configs:
            return

        # This may look weird, but it's here for defensive purposes.
        # Currently, the possible values for CircleCI's "vcs-type" field in
        # their API happens to match up perfectly with our service names,
        # but that's not necessarily always going to be the case.
        if service_name == 'github':
            vcs_type = 'github'
        elif service_name == 'bitbucket':
            vcs_type = 'bitbucket'
        else:
            raise ValueError('Unexpected hosting service type got through '
                             'to CircleCI invocation: %s'
                             % service_name)

        org_name, repo_name = self._get_repo_ids(service_name, repository)

        user = self._get_or_create_user()

        for config in matching_configs:
            status_update = StatusUpdate.objects.create(
                service_id='circle-ci',
                user=user,
                summary='CircleCI',
                description='starting build...',
                state=StatusUpdate.PENDING,
                review_request=review_request,
                change_description=changedesc)

            url = ('https://circleci.com/api/v1.1/project/%s/%s/%s/tree/%s'
                   '?circle-token=%s'
                   % (vcs_type, org_name, repo_name,
                      config.get('branch_name') or 'master',
                      urlquote_plus(config.get('circle_api_token'))))

            logger.info('Making CircleCI API request: %s', url)

            local_site = config.local_site

            try:
                token = user.webapi_tokens.filter(local_site=local_site)[0]
            except IndexError:
                token = WebAPIToken.objects.generate_token(
                    user, local_site=local_site, autogenerated=True)

            body = {
                'revision': diffset.base_commit_id,
                'build_parameters': {
                    'CIRCLE_JOB': 'reviewboard',
                    'REVIEWBOARD_SERVER':
                        get_server_url(local_site=config.local_site),
                    'REVIEWBOARD_REVIEW_REQUEST': review_request.display_id,
                    'REVIEWBOARD_DIFF_REVISION': diffset.revision,
                    'REVIEWBOARD_API_TOKEN': token.token,
                    'REVIEWBOARD_STATUS_UPDATE_ID': status_update.pk,
                },
            }

            if config.local_site:
                body['build_parameters']['REVIEWBOARD_LOCAL_SITE'] = \
                    config.local_site.name

            request = URLRequest(
                url,
                body=json.dumps(body),
                headers={
                    'Accept': 'application/json',
                    'Content-Type': 'application/json',
                },
                method='POST')

            u = urlopen(request)

            data = json.loads(u.read())

            status_update.url = data['build_url']
            status_update.url_text = 'View Build'
            status_update.save()

    def _get_repo_ids(self, service_name, repository):
        """Return the organization and repo name for the given repository.

        Args:
            repository (reviewboard.scmtools.models.Repository):
                The repository.

        Returns:
            tuple of unicode:
            A two-tuple consisting of the organization (or user) and repository
            names.
        """
        extra_data = repository.extra_data
        plan = extra_data['repository_plan']

        if service_name == 'github':
            if plan == 'public':
                return (extra_data['hosting_account_username'],
                        extra_data['github_public_repo_name'])
            elif plan == 'public-org':
                return (extra_data['github_public_org_name'],
                        extra_data['github_public_org_repo_name'])
            elif plan == 'private':
                return (extra_data['hosting_account_username'],
                        extra_data['github_private_repo_name'])
            elif plan == 'private-org':
                return (extra_data['github_private_org_name'],
                        extra_data['github_private_org_repo_name'])
            else:
                raise ValueError('Unexpected plan for GitHub repository %d: %s'
                                 % (repository.pk, plan))
        elif service_name == 'bitbucket':
            if plan == 'personal':
                return (extra_data['bitbucket_account_username'],
                        extra_data['bitbucket_repo_name'])
            elif plan == 'other-user':
                return (extra_data['bitbucket_other_user_username'],
                        extra_data['bitbucket_other_user_repo_name'])
            elif plan == 'team':
                return (extra_data['bitbucket_team_name'],
                        extra_data['bitbucket_team_repo_name'])
            else:
                raise ValueError('Unexpected plan for Bitbucket repository '
                                 '%d: %s'
                                 % (repository.pk, plan))

    def _get_or_create_user(self):
        """Return a user to use for CircleCI.

        Returns:
            django.contrib.auth.models.User:
            A user instance.
        """
        try:
            return User.objects.get(username='circle-ci')
        except User.DoesNotExist:
            logger.info('Creating new user for CircleCI')
            siteconfig = SiteConfiguration.objects.get_current()
            noreply_email = siteconfig.get('mail_default_from')

            with transaction.atomic():
                try:
                    user = User.objects.create(username='circle-ci',
                                               email=noreply_email,
                                               first_name='Circle',
                                               last_name='CI')
                except IntegrityError:
                    # Another process/thread beat us to it.
                    return User.objects.get(username='circle-ci')

                profile = user.get_profile()
                profile.should_send_email = False
                profile.save()

                avatar_service = avatar_services.get_avatar_service(
                    URLAvatarService.avatar_service_id)
                # TODO: make somewhat higher-res versions for the main avatar.
                avatar_service.setup(user, self.icon_static_urls)

                return user


#!/usr/bin/env python3
# coding: utf-8

import os
import grpc
import logging
import numpy as np

import jarvis_api.audio_pb2 as ja
import jarvis_api.jarvis_tts_pb2 as jtts
import jarvis_api.jarvis_tts_pb2_grpc as jtts_srv

from jetson_voice import TTSService

class JarvisTTSService(TTSService):
    """
    Jarvis streaming TTS service.
    """
    def __init__(self, config, *args, **kwargs):
        """
        Open a streaming channel to the Jarvis server for TTS.  This establishes a connection over GRPC
        and sends/recieves the requests and responses.
        """
        super(JarvisTTSService, self).__init__(config, *args, **kwargs)

        self.config.setdefault('server', 'localhost:50051')
        self.config.setdefault('sample_rate', 22050)        # ignored (will always be 22.05KHz)
        self.config.setdefault('voice_name', 'ljspeech')    # ignored
        self.config.setdefault('language_code', 'en-US')

        logging.info(f'Jarvis TTS service config:\n{self.config}')

        self.channel = grpc.insecure_channel(self.config.server)
        self.client = jtts_srv.JarvisTTSStub(self.channel)

    def __call__(self, text):
        """
        Generate audio from text.

        Parameters:
          text (string) -- The phrase to convert to audio.

        Returns audio samples in a numpy array.
        """
        req = jtts.SynthesizeSpeechRequest()

        req.text = text
        req.language_code = self.config.language_code
        req.sample_rate_hz = self.config.sample_rate
        req.voice_name = self.config.voice_name
        req.encoding = ja.AudioEncoding.LINEAR_PCM

        resp = self.client.Synthesize(req)

        samples = np.frombuffer(resp.audio, dtype=np.float32)
        return samples

    @property
    def sample_rate(self):
        """
        Get the output sample rate (in Hz)
        """
        return self.config.sample_rate


#!/usr/bin/env python2

import time
import threading
import csv
import numpy as np
from multiprocessing import Process
import sys
import serial
from sklearn.externals import joblib
from scipy import signal

sys.path.append('..')
from open_bci import *
from pylab import *

#'/dev/tty.usbmodemfd121'
#Notes for using csv_collector.py
#initiate CSVCollector, takes filename, port, and baud as inputs
#start recording with start() method
#tag recording with tag() method
#what needs to be implemented
#   A if statement for multiprocessing/threading.

OPENBCI_PORT = '/dev/ttyACM0'
TEENSY_PORT = '/dev/ttyACM1'
TEENSY_ENABLED = False

box_width = 250
look_back = 60
M = 25
r = 250
f1, f2 = 7, 14

wavelet1 = signal.morlet(M, w=(f1*M)/(2.0*r))
wavelet2 = signal.morlet(M, w=(f2*M)/(2.0*r))

def extract_features(data):
    sigs = np.zeros((data.shape[0], 3))
    sigs[..., 0] = (data[..., 0] + data[..., 1])/2.0
    sigs[..., 1] = (data[..., 2] + data[..., 3])/2.0
    sigs[..., 2] = (data[..., 4] + data[..., 5])/2.0

    # fft_len = np.fft.rfft(data[..., 0]).shape[0]
    features = np.array([])

    for j in range(3):
        sig = sigs[..., j]
        conv1 = signal.convolve(sig, wavelet1, 'same')
        conv2 = signal.convolve(sig, wavelet2, 'same')
        fourier = np.fft.fft(sig)
        fourier1 = np.fft.fft(conv1)
        fourier2 = np.fft.fft(conv2)
        features = np.hstack([features, np.abs(fourier), np.abs(fourier1), np.abs(fourier2)])
        # not sure if this is a good idea -->
        features = np.hstack([features, np.angle(fourier), np.angle(fourier1), np.angle(fourier2)])

    return features

class MIPlotter(object):

    def __init__(self, port=None, baud=115200):
        print("connecting to OpenBCI...")
        self.board = OpenBCIBoard(port, baud)

        self.bg_thread = None
        self.bg_draw_thread = None
        self.data = np.array([0]*8)
        self.should_plot = False
        self.control = np.array([0,0,0])
        self.control_f = np.array([0])
        self.out_sig = np.array([0])
        self.controls = np.array([[0]*4])
        self.eye_r = np.array([0])
        self.eye_l = np.array([0])

        model, good_features = joblib.load('neighbors_model.pkl')
        # self.eye_l_temp, self.eye_r_temp = joblib.load('eye_blinks.pkl')
        self.good_features = good_features
        self.model = model

        print("connecting to teensy...")
        if TEENSY_ENABLED:
            self.teensy = serial.Serial(TEENSY_PORT, 57600)

    def stop(self):
        # resolve files and stuff
        self.board.should_stream = False
        self.should_plot = False
        #self.bg_thread.join()
        self.bg_thread = None
        self.data = np.array([0]*8)

    def disconnect(self):
        self.board.disconnect()

    def plot(self):

        plt.clf()

        hold(True)
        data = np.copy(self.data)

        control_arr = np.zeros(look_back)

        for i in range(look_back):
            d = data[i:(box_width+i)]
            features = extract_features(d)
            features = features[self.good_features]
            control_arr[i] = self.model.predict(features)

        control = control_arr.mean()
        control_f = 0.1 * control + 0.8 * self.control[-1]

        # control_f2 = 1 * control_f - 1 * self.control[-1]
        # control_f3 = 0.6 * control_f2 + 0.4 * self.control_f[-1]

        self.control = np.append(self.control, control_f)
        # self.control_f = np.append(self.control_f, control_f3)
        # self.controls = np.vstack([self.controls, controls_f])

        # out_sig = control

        if control_f < -0.07:
            out_sig = 550
        elif control_f > 0.07:
            out_sig = 450
        else:
            out_sig = 500

        # r_max = np.max(signal.correlate(self.eye_r_temp, data[..., 6]))
        # l_max = np.max(signal.correlate(self.eye_l_temp, data[..., 7]))

        # self.eye_r = np.append(self.eye_r, r_max)
        # self.eye_l = np.append(self.eye_l, l_max)

        self.out_sig = np.append(self.out_sig, out_sig)

        if TEENSY_ENABLED:
            self.teensy.write("0" + str(out_sig) + "\r")

        # for i in range(4):
        #     plot(self.controls[-40:, i], label=str(i+1))

        # plot(sig)
        # plot(conv1)
        # plot(conv2)

        plot(self.control[-40:])
        # plot(self.out_sig[-40:] * 0.001)

        # plot(self.eye_l[-40:])
        # plot(self.eye_r[-40:])
        # plot((self.eye_r[-40:] > 0.00011) * 0.001)

        # plot(self.control_f[-40:])
        # plot(self.out_sig[-40:] * 0.001)
        # ylim([-0.005, 0.005])

        # ylim([0, 0.02])

        # legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
        #        ncol=4, mode="expand", borderaxespad=0.)

            #plot(freq, np.log(abs(fourier)), label=str(i+1))

            #title('channel {0}'.format(i+1))
        # ylim([-0.0005, 0.0005])
        # ylim([-12, 0])
        # xlim([0, 60])
        # legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
        #        ncol=4, mode="expand", borderaxespad=0.)

        # plot(freq, np.log(abs(fourier)))

        show(block=False)
        draw()

    def background_plot(self):
        while self.should_plot:
            if len(self.data) >= box_width + look_back:
                self.plot()
            time.sleep(0.05)

    def receive_sample(self, sample):
        t = time.time()
        idd = sample.id
        sample = sample.channels
        if not np.any(np.isnan(sample)):
            self.data = np.vstack( (self.data[-250-look_back:, ...], sample) )

    def start(self):

        if self.bg_thread:
            self.stop()

        #create a new thread in which the OpenBCIBoard object will stream data
        self.bg_thread = threading.Thread(target=self.board.start,
                                        args=(self.receive_sample, ))
        # self.bg_thread = Process(target=self.board.start,
        #                         args=(self.receive_sample, ))

        self.bg_thread.start()

        # self.bg_draw_thread = threading.Thread(target=self.background_plot,
        #                                        args=())

        # self.bg_draw_thread.start()

        ion()
        figure()
        show(block=False)

        self.should_plot = True

        self.background_plot()

if __name__ == '__main__':
    plotter = MIPlotter(port=OPENBCI_PORT)
    plotter.start()

    plt.rc('axes', color_cycle=['red', 'orange', 'yellow', 'green'])


"""A setuptools based setup module.

See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""

# To use a consistent encoding
from codecs import open
from os import path

# Always prefer setuptools over distutils
from setuptools import find_packages, setup

from telethon import TelegramClient

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='Telethon',

    # Versions should comply with PEP440.
    version=TelegramClient.__version__,
    description="Python3 Telegram's client implementation with full access to its API",
    long_description=long_description,

    # The project's main homepage.
    url='https://github.com/LonamiWebs/Telethon',
    download_url='https://github.com/LonamiWebs/Telethon/releases',

    # Author details
    author='Lonami Exo',
    author_email='totufals@hotmail.com',

    # Choose your license
    license='MIT',

    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        # How mature is this project? Common values are
        #   3 - Alpha
        #   4 - Beta
        #   5 - Production/Stable
        'Development Status :: 3 - Alpha',

        # Indicate who your project is intended for
        'Intended Audience :: Developers',
        'Topic :: Communications :: Chat',

        # Pick your license as you wish (should match "license" above)
        'License :: OSI Approved :: MIT License',

        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
    ],

    # What does your project relate to?
    keywords='Telegram API chat client MTProto',

    # You can just specify the packages manually here if your project is
    # simple. Or you can use find_packages().
    packages=find_packages(exclude=[
        'telethon_generator', 'telethon_tests', 'run_tests.py',
        'try_telethon.py'
    ]),

    # List run-time dependencies here. These will be installed by pip when
    # your project is installed.
    install_requires=['pyaes'],

    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # pip to create the appropriate form of executable for the target platform.
    entry_points={
        'console_scripts': [
            'gen_tl = tl_generator:clean_and_generate',
        ],
    })


import os
import pdb
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# import sys

def pickle_data(env_data):
	pass

def load_env(env_string):
	if env_string == 'eth':
		peds_data1, obs_data1 = load_eth_env('../crowd_dataset/ETH/ewap_dataset/seq_eth/')
		peds_data2, obs_data2 = load_eth_env('../crowd_dataset/ETH/ewap_dataset/seq_hotel/')
		return ((peds_data1, peds_data2), (obs_data1, obs_data2))
	elif env_string == 'ucy':
		ped_data1 = load_ucy_env('../crowd_dataset/UCY/arxie/')
		ped_data2 = load_ucy_env('../crowd_dataset/UCY/university/')
		ped_data3 = load_ucy_env('../crowd_dataset/UCY/zara/')
		return peds_data1, ped_data2, ped_data3
	elif env_string == 'grand':
		ped_data = load_grand_env('../crowd_dataset/grand-central/')
		return ped_data
	elif env_string == 'train':
		ped_data = load_train_env('../crowd_dataset/train-station/')
		return ped_data
	elif env_string == 'all':
		eth_data1, _ = load_eth_env('../crowd_dataset/ETH/ewap_dataset/seq_eth/')
		eth_data2, _ = load_eth_env('../crowd_dataset/ETH/ewap_dataset/seq_hotel/')
		ucy_data1 = load_ucy_env('../crowd_dataset/UCY/arxie/')
		ucy_data2 = load_ucy_env('../crowd_dataset/UCY/university/')
		ucy_data3 = load_ucy_env('../crowd_dataset/UCY/zara/')
		gra_data = load_grand_env('../crowd_dataset/grand-central/')
		tra_data = load_train_env('../crowd_dataset/train-station/')

'''
GRAND CENTRAL CROWD DATASET PROCESSING
'''
def load_grand_env(directory):
	pass

'''
TRAIN CROWD DATASET PROCESSING
'''
def load_train_env(directory):
	pass

'''
UCY CROWD DATASET PROCESSING
'''
def load_ucy_env(directory):
	files = os.listdir(directory)
	for file in files:
		file = directory + file
		with open(file, 'r') as file:
			data = file.read().split('\n')
			pad.set_trace()

'''
ETH CROWD DATASET PROCESSING
Note: This dataset is presented in meters (m)
When training on this dataset individually this is fine; however, when
using all the datasets as a training set this dataset must be converted
to pixel units as the remain datasets are in the image plane.

Q: Does the origin shift shifting matter? i.e. UCY origin is center of image plane
while train origin is left top corner.
A: Yes. This is solved with a simple translation based on the image size.
'''
def load_eth_env(directory, pixels=False):
	ped_file = directory + 'obsmat.txt'
	obs_file = directory + 'map.png'
	H_file = directory + 'H.txt'
	peds_data = load_eth_pedestrian(ped_file, H_file, pixels=pixels)
	obs_data = load_eth_obstacles(obs_file, H_file, pixels)
	return peds_data, obs_data

def load_eth_pedestrian(file, h_file, pixels=False):
	if pixels:
		with open(h_file, 'r') as file:
			H = file.read().split('\n')
			for i, h in enumerate(H):
				h = np.array(h.split(' '))
				h_indices = np.where(h != '')
				H[i] = [float(i) for i in h[h_indices]]

		H = np.array(H[:-1])

	with open(file, 'r') as file:
		peds_data = file.read().split('\n')
		peds_frame = []
		data = []
		frame = 1
		for i, ped in enumerate(peds_data):
			ped_data = np.array(ped.split(' '))
			indices = np.where(ped_data != '')
			n_ped_data = np.array([float(j) for j in ped_data[indices]])
			if len(n_ped_data) == 0: continue
			if n_ped_data[0] == frame:
				peds_frame.append(n_ped_data)
			else:
				frame = n_ped_data[0]
				peds_frame_np = np.array(peds_frame)
				data.append(peds_frame_np)
				peds_frame.clear()
				peds_frame.append(n_ped_data)

			# peds_data[i] = np.array(n_ped_data)

		# peds_data = peds_data[:-1] # Remove last empty line
		# peds_data[i] = ped_data
		# pdb.set_trace()
	return np.array(data)

def load_eth_obstacles(map, h_file, border=False):
	img = mpimg.imread(map)
	n,m = np.shape(img)

	if border:
		# Outer Edge of Image
		x_border = np.arange(n).reshape((1,-1))
		y_border = np.arange(m).reshape((1,-1))
		x0 = np.vstack((x_border, np.zeros((1,n))))
		xm = np.vstack((x_border, m*np.ones((1,n))))
		y0 = np.vstack((np.zeros((1,m)), y_border))
		yn = np.vstack((n*np.ones((1,m)), y_border))
		borders = np.hstack((x0,xm,y0,yn))
		_, m = np.shape(borders)

	indices = np.where(img == 1.)
	_, n = np.shape(indices)

	if border:
		obstacles = np.hstack((indices, borders))
		obstacles = np.vstack((obstacles, np.ones((1,n+m))))
	else:
		obstacles = np.vstack((indices, np.ones((1,n))))

	with open(h_file, 'r') as file:
		H = file.read().split('\n')
		for i, h in enumerate(H):
			h = np.array(h.split(' '))
			h_indices = np.where(h != '')
			H[i] = [float(i) for i in h[h_indices]]

	H = np.array(H[:-1])
	obs = H @ obstacles
	return obs

# Re-organizes data in terms inputs and outputs
# Organize in tensor of 'm x N x T' where m is the dimension, N are the agents, T is the time
def generate_eth_training_data(ped_data, inspace, outspace): # pos : position | vel : velocity | pv : pos & vel
	inputs = []
	outputs = []
	# pdb.set_trace()
	if inspace == 'pos' and outspace == 'vel': # Input: Position, Output: Current velocity
		for i, frame in enumerate(ped_data):
			for ped in frame:
				inputs.append([ped[2], ped[4]])
				outputs.append([ped[5], ped[7]])

		inputs = np.array(inputs)
		outputs = np.array(outputs)
		return inputs, outputs

	elif inspace == 'pos' and outspace == 'pos': # Input: Position, Output: Next timesteps position
		# Must grab the position from the next frame if it exists.
		for i, frame in enumerate(ped_data):
			for ped in frame:
				if i < len(ped_data) - 1:
					for ped_future in ped_data[i + 1]:
						if ped[1] == ped_future[1]:
							inputs.append([ped[2], ped[4]])
							outputs.append([ped_future[2], ped_future[4]])

		inputs = np.array(inputs)
		outputs = np.array(outputs)
		return inputs, outputs

	elif inspace == 'pv' and outspace == 'pos': # Input: Position & Velocity, Output: Next timesteps position
		for i, frame in enumerate(ped_data):
			for ped in frame:
				if i < len(ped_data) - 1:
					for ped_future in ped_data[i + 1]:
						if ped[0] == ped_future[0]:
							inputs.append([ped[2], ped[4], ped[5], ped[7]])
							outputs.append([ped_future[2], ped_future[4]])

		inputs = np.array(inputs)
		outputs = np.array(outputs)
		return inputs, outputs

	elif inspace == 'pv' and outspace == 'vel': # Input: Position & Velocity, Output: Next timesteps velocity
		for i, frame in enumerate(ped_data):
			for ped in frame:
				if i < len(ped_data) - 1:
					for ped_future in ped_data[i + 1]:
						if ped[0] == ped_future[0]:
							inputs.append([ped[2], ped[4], ped[5], ped[7]])
							outputs.append([ped_future[5], ped_future[7]])

		inputs = np.array(inputs)
		outputs = np.array(outputs)
		return inputs, outputs

	elif inspace == 'pv' and outspace == 'pv': # Input: Position & Velocity, Output: Next position & velocity
		for i, frame in enumerate(ped_data):
			for ped in frame:
				if i < len(ped_data) - 1:
					for ped_future in ped_data[i + 1]:
						if ped[0] == ped_future[0]:
							inputs.append([ped[2], ped[4], ped[5], ped[7]])
							outputs.append([ped_future[2], ped_future[4], ped_future[5], ped_future[7]])

		inputs = np.array(inputs)
		outputs = np.array(outputs)
		return inputs, outputs


#!/usr/bin/env python
# coding=utf-8

# Copyright [2020] [Apache Software Foundation]
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import os.path
import os
import sys

from setuptools import setup, find_packages
from setuptools.command.test import test as TestCommand

# Package basic info
PACKAGE_NAME = 'marvin_python_toolbox'
PACKAGE_DESCRIPTION = 'Apache Marvin-AI Python Toolbox'

URL = ''

AUTHOR_NAME = 'Apache Marvin-AI Community'
AUTHOR_EMAIL = 'dev@marvin.apache.org'

PYTHON_2 = False
PYTHON_3 = True

# Project status
# (should be 'planning', 'pre-alpha', 'alpha', 'beta', 'stable', 'mature' or 'inactive').
STATUS = 'planning'

# Project topic
# See https://pypi.python.org/pypi?%3Aaction=list_classifiers for a list
TOPIC = 'Topic :: Software Development :: Libraries :: Python Modules',

# External dependencies
# More info https://pythonhosted.org/setuptools/setuptools.html#declaring-dependencies
REQUIREMENTS_EXTERNAL = [
    'docker',
    'configparser',
    'bump2version',
    'wget',
    'grpcio',
    'grpcio-tools',
    'click',
    'cookiecutter',
    'docker-compose',
    'matplotlib',
    'cryptography',
    'kubernetes',
    'requests'
]

# This is normally an empty list
DEPENDENCY_LINKS_EXTERNAL = []
# script to be used
SCRIPTS = ['bin/marvin',
            'bin/marvin_complete',
            'bin/marvin-api']

def _get_version():
    """Return the project version from VERSION file."""
    with open(os.path.join(os.path.dirname(__file__), PACKAGE_NAME, 'VERSION'), 'rb') as f:
        version = f.read().decode('ascii').strip()
    return version

DEVELOPMENT_STATUS = {
    'planning': '1 - Planning',
    'pre-alpha': '2 - Pre-Alpha',
    'alpha': 'Alpha',
    'beta': '4 - Beta',
    'stable': '5 - Production/Stable',
    'mature': '6 - Mature',
    'inactive': '7 - Inactive',
}

CLASSIFIERS = ['Development Status :: {}'.format(DEVELOPMENT_STATUS[STATUS])]
if PYTHON_2:
    CLASSIFIERS += [
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
    ]
if PYTHON_3:
    CLASSIFIERS += [
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.6',
    ]

setup(
    name=PACKAGE_NAME,
    version=_get_version(),
    url=URL,
    description=PACKAGE_DESCRIPTION,
    long_description=open(os.path.join(
        os.path.dirname(__file__), 'README.md')).read(),
    author=AUTHOR_NAME,
    maintainer=AUTHOR_NAME,
    maintainer_email=AUTHOR_EMAIL,
    packages=find_packages(exclude=('tests', 'tests.*')),
    include_package_data=True,
    zip_safe=False,
    classifiers=CLASSIFIERS,
    install_requires=REQUIREMENTS_EXTERNAL,
    dependency_links=DEPENDENCY_LINKS_EXTERNAL,
    scripts=SCRIPTS,
)


from setuptools import setup

__VERSION__ = "0.3.1"

setup(
    name="joseph_hello_world",
    description="A hello world plugin for Joseph. Write an hello world message to the logs.",
    author="Niek Keijzer",
    author_email="info@niekkeijzer.com",
    url="https://github.com/NiekKeijzer/HelloWorld",
    download_url="https://github.com/NiekKeijzer/HelloWorld/archive/{}.tar.gaz".format(__VERSION__),
    keywords=[
        "hello world",
        "joseph"
    ],
    packages=[
        "hello_world"
    ],
    version=__VERSION__,
    entry_points={
        'joseph.actions': [
            'say_hello = hello_world.hello:say_hello'
        ]
    }
)


# -*- coding: utf-8 -*-

from base.mailbox import MailboxManager
from models.sender_account import SenderAccountDao

class SenderAccountManager(MailboxManager):

    """
    Lógica de negocio correspondiente a la entidad SenderAccountDao.
    """

    def __init__(self, user=None, sender_account_id=None):

        """
        Constructor de la clase.
        :param user: Usuario actual.
        :param sender_account_id: Identificador del buzón emisor.
        """

        super(SenderAccountManager, self).__init__(user)
        if sender_account_id:
            self._sender_account_id = int(sender_account_id)

    def create(self, sender_account):

        """
        Crea un buzón emisor.
        :param sender_account: Diccionario que representa un buzón emisor.
        :return: El buzón emisor creado.
        """

        from oauth2client.contrib.appengine import CredentialsModel

        # Comprobamos que los campos obligatorios están incluidos.
        if "email" not in sender_account or not sender_account["email"]:
            raise Exception("The account must have an email.")

        # Volcamos el contenido de sender_account y actualizamos usuario creador.
        entity = SenderAccountDao(**sender_account)
        entity.created_by = self._user
        entity.updated_by = self._user
        # Comprobamos si el buzón ya ha autorizado el acceso de la aplicación.
        # En ese caso, lo establecemos como autorizado.
        entity.is_authorized = CredentialsModel.get_by_key_name(entity.email) is not None
        entity.put()
        return entity

    def get(self):

        """
        Obtiene un buzón emisor.
        :return: El buzón emisor obtenido.
        """

        from google.appengine.ext.ndb import Key

        entity = Key(SenderAccountDao, self._sender_account_id).get()
        # Comprobamos si el buzón existe.
        if entity is None:
            raise Exception("The given entity does not exist.")
        return entity

    @classmethod
    def list(cls):

        """
        Obtiene la colección de buzones emisores.
        :return: La colección de buzones obtenida.
        """

        from google.appengine.api import memcache

        memcache_key = SenderAccountDao.COLLECTION_MEMCACHE_KEY
        entities = memcache.get(memcache_key)

        if entities is None:
            entities = [k.to_dict() for k in SenderAccountDao.query().iter()]
            memcache.add(memcache_key, entities, 0)
        return entities

    def update(self, sender_account):

        """
        Actualiza el buzón emisor indicado.
        :param sender_account: Diccionario que representa el buzón emisor.
        :return: El buzón emisor actualizado.
        """

        entity = self.get()
        # Comprobamos que campos hay que actualizar.
        if "email" in sender_account and sender_account["email"]:
            entity.email = sender_account["email"]
        if "group" in sender_account and sender_account["group"]:
            entity.group = sender_account["group"]
        if "is_active" in sender_account and isinstance(sender_account["is_active"], bool):
            entity.is_active = sender_account["is_active"]
        entity.updated_by = self._user
        entity.put()
        return entity

    def add_group(self, group_id):

        """
        Agrega el buzón emisor a un nuevo grupo.
        :param group_id: Identificador del grupo.
        :return: La cuenta actualizada.
        """

        from models.master import MasterDao
        from managers.group import GroupManager

        entity = self.get()
        group_id = int(group_id)
        # Comprobamos que el buzón emisor no pertenezca actualmente al grupo dado.
        if filter(lambda x: x.id == group_id, entity.groups):
            raise Exception("This account is already in this group.")
        # En caso contrario, lo actualizamos.
        group_entity = GroupManager(group_id=group_id).get()
        master = MasterDao()
        master.id = group_entity.key.id()
        master.name = group_entity.name
        entity.groups.append(master)
        entity.updated_by = self._user
        entity.put()
        return entity

    def delete(self):

        """
        Elimina el buzón emisor dado.
        """

        from google.appengine.ext.ndb import Key

        Key(SenderAccountDao, self._sender_account_id).delete()

    def delete_group(self, group_id):

        """
        Elimina un grupo de la lista de grupos del buzón emisor.
        :param group_id: Identificador del grupo.
        :return: La cuenta actualizada.
        """

        entity = self.get()
        found = False
        group_id = int(group_id)
        for i in range(len(entity.groups)):
            if entity.groups[i].id == group_id:
                entity.groups.pop(i)
                found = True
                break
        # Comprobamos que el buzón emisor pertenezca al grupo dado.
        if not found:
            raise Exception("This account is not in this group.")
        entity.put()
        return entity

    def available_groups(self):

        """
        Obtiene la colección de grupos a los cuales pertenece el buzón emisor.
        :return: La colección obtenida.
        """

        from managers.group import GroupManager

        entity = self.get()
        taken_groups = set([k.id for k in entity.groups])
        entity_groups = GroupManager().list()
        all_groups = set([k["id"] for k in entity_groups])
        # [k for k in entity_groups if k["id"] in list(all_groups.difference(taken_groups))]
        return filter(lambda x: x["id"] in list(all_groups.difference(taken_groups)), entity_groups)

    @staticmethod
    def get_by_email(email):

        """
        Obtiene un buzón receptor.
        :return: El buzón receptor obtenido.
        """

        entity = SenderAccountDao.query(SenderAccountDao.email == email).fetch(1)
        # Comprobamos que el buzón existe actualmente.
        if entity is None:
            raise Exception("The given entity does not exist.")
        return entity

    def authorize(self, email):

        """
        Establece el buzón receptor dado como autorizado, crea las etiquetas DUPLICADO,
        GESTIONADO, PDTE REINTENTAR y ERROR en el buzón dado.
        :param email: Identificador del buzón.
        :return: El buzón autorizado.
        """

        from pending_authorization import PendingAuthorizationManager
        from core.logger import Logger

        try:
            entity = self.get_by_email(email)
            if entity is not None:
                # Marcamos el buzón como autorizado.
                entity.is_authorized = True
                # Añadimos la información de tracking.
                entity.updated_by = self._user
                entity.put()
                # Obtenemos el diccionario que representa el buzón actualizado.
                entity = entity.to_dict()
                # Eliminamos la autorización.
                PendingAuthorizationManager.delete(entity["user_id"])
        except Exception as e:
            Logger.error(e)
            raise e
        return entity