# -*- coding: utf-8 -*-
"""
Created on Fri Apr  5 15:23:01 2013

Finite Difference Approximation and Convergence

@author: jspineda
"""
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from array_op import rotate #use np.roll

#interval [-2,6]

def f(xin):
    "Function f = x3 - 5x2 + x"
    x = np.array(xin)
    return np.power(x,3) - 5*np.power(x,2) + x

def df_sol(xin):
    "Analytic Derivative of f, see above"
    x = np.array(xin)
    return 3*np.power(x,2) - 10*x + 1

def df_for(x):
    "Forward derivative at x, using input descritization, ouput array has n-1"
    h = (rotate(x,-1) - x)[1]
    return ( f(rotate(x,-1)) - f(x))[:-1] / h  #does not include the latter edge point

def df_cent(x):
    "Central derivative at x, in step size h"
    h = (rotate(x,-1) - x)[1]
    return (f(rotate(x,-1)) - f(rotate(x,1)))[1:-1] / (2*h) #does not include edge points

def absicc(h,interval):
    "produce absicca on interval [-2,6] with given stepsize"
    n = int((interval[1] - interval[0])/ h)
    return h*np.arange(0,n+1) - 2

def quest(h):
    "Command for executing code for problem 1"
    interval = [-2,6.]
    x1 = absicc(h,interval) # define x axis on the interval with resolution h
    xf1 = x1[:-1]
    xc1 = x1[1:-1]
    x2 = absicc(h/2.,interval) # define x axis on the interval with resolution h/2
    xf2 = x2[:-1]
    xc2 = x2[1:-1]

    plt.clf()
    fig1 = plt.figure(1)
    plt.plot(xf1,df_for(x1) - df_sol(xf1),color='orange',label='Step Size: {0}'.format(h))
    plt.plot(xf2,df_for(x2) - df_sol(xf2),color='red',label='Step Size: {0}'.format(h/2.))
    plt.legend(loc='lower right')
    plt.title('Forward Differencing')
    pp1 = PdfPages('ws2_prob1_fig1.pdf')
    pp1.savefig(fig1)
    pp1.close()

    plt.clf()
    fig2 = plt.figure(2)
    plt.plot(xc1,df_cent(x1) - df_sol(xc1),color='darkgreen',label='Step Size: {0}'.format(h))
    plt.plot(xc2,df_cent(x2) - df_sol(xc2),color='blue',label='Step Size: {0}'.format(h/2))
    plt.legend(loc='center right')
    plt.title('Central Differencing')
    pp2 = PdfPages('ws2_prob1_fig2.pdf')
    pp2.savefig(fig2)
    pp2.close()

    return 'Done, see plots'


"""Read analytics from local SQL files."""
from os import listdir
from os.path import isfile, join
from typing import List

from sqlalchemy.engine.result import ResultProxy

from config import basedir
from database import rdbms

def collect_sql_queries(subdirectory: str) -> dict:
    """
    Create dict of SQL queries to be run where `keys` are filenames and `values` are queries.

    :param subdirectory: Directory containing .sql queries to run in bulk.
    :type subdirectory: str
    :returns: dict
    """
    sql_file_paths = fetch_sql_files(subdirectory)
    sql_queries = parse_sql_batch(sql_file_paths)
    sql_file_names = [file.split("/")[-1] for file in sql_file_paths]
    query_dict = dict(zip(sql_file_names, sql_queries))
    return query_dict

def fetch_sql_files(subdirectory: str) -> List[str]:
    """
    Fetch all SQL query files in folder.

    :param subdirectory: Subdirectory containing SQL files to fetch.
    :type subdirectory: str
    :returns: List[str]
    """
    folder = f"{basedir}/database/queries/{subdirectory}"
    directory = listdir(folder)
    files = [
        folder + "/" + f for f in directory if isfile(join(folder, f)) if ".sql" in f
    ]
    return files

def parse_sql_batch(sql_file_paths: List[str]) -> List[str]:
    """
    Read SQL analytics from .sql files.

    :param sql_file_paths: List of paths to SQL files to read and parse.
    :type sql_file_paths: List[str]
    :returns: List[str]
    """
    queries = []
    for file in sql_file_paths:
        sql_file = open(file, "r")
        query = sql_file.read()
        queries.append(query)
        sql_file.close()
    return queries

def fetch_raw_lynx_posts() -> ResultProxy:
    """
    Find all Lynx posts lacking embedded link previews.

    :returns: ResultProxy
    """
    sql_file = open(f"{basedir}/database/queries/posts/selects/lynx_bookmarks.sql", "r")
    query = sql_file.read()
    posts = rdbms.execute_query(query, "hackers_prod").fetchall()
    return posts


from numpy.lib.arraysetops import isin
from numpy.lib.function_base import average
from sklearn.metrics import mean_squared_error, mean_absolute_error, f1_score, r2_score
from sklearn.metrics import precision_score, recall_score
import numpy as np
import pandas as pd

def mae(app_gt,app_pred):
    return mean_absolute_error(app_gt,app_pred)

def rmse(app_gt, app_pred):
    return mean_squared_error(app_gt,app_pred)**(.5)

def f1score(app_gt, app_pred):
    threshold = 10
    gt_temp = np.array(app_gt)
    gt_temp = np.where(gt_temp<threshold,0,1)
    pred_temp = np.array(app_pred)
    pred_temp = np.where(pred_temp<threshold,0,1)

    return f1_score(gt_temp, pred_temp)

def f1score_(app_gt, app_pred):
    threshold = 10
    gt_temp = np.array(app_gt)
    gt_temp = np.where(gt_temp<threshold,0,1)
    pred_temp = np.array(app_pred)
    pred_temp = np.where(pred_temp<threshold,0,1)

    return f1_score(gt_temp, pred_temp, average='macro')

def recall(app_gt, app_pred):
    threshold = 10
    gt_temp = np.array(app_gt)
    gt_temp = np.where(gt_temp<threshold,0,1)
    pred_temp = np.array(app_pred)
    pred_temp = np.where(pred_temp<threshold,0,1)

    return recall_score(gt_temp, pred_temp, )

def precision(app_gt, app_pred):
    threshold = 10
    gt_temp = np.array(app_gt)
    gt_temp = np.where(gt_temp<threshold,0,1)
    pred_temp = np.array(app_pred)
    pred_temp = np.where(pred_temp<threshold,0,1)

    return precision_score(gt_temp, pred_temp, )

def relative_error(app_gt,app_pred):
    constant = 1
    numerator = np.abs(app_gt - app_pred)
    denominator = constant + app_pred
    return np.mean(numerator/denominator)

def r2score(app_gt,app_pred):
    # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.r2_score.html#sklearn.metrics.r2_score
    return r2_score(app_gt, app_pred)

def nde(app_gt,app_pred):
    # Normalized Disaggregation Error (NDE)
    # Inspired by http://proceedings.mlr.press/v22/zico12/zico12.pdf
    numerator = np.sum((app_gt-app_pred)**2)
    denominator = np.sum(app_gt**2)

    # if denominator < 1e-5:
    #     denominator = 1e-5

    frac = numerator/denominator
    if isinstance(frac, pd.Series):
        return np.sqrt(frac[0])
    else:
        return np.sqrt(frac)

def nep(app_gt,app_pred):
    # Normalized Error in Assigned Power (NEP)
    # Inspired by https://www.springer.com/gp/book/9783030307813
    numerator = np.sum(np.abs(app_gt-app_pred))
    denominator = np.sum(app_gt)

    # if denominator < 1e-5:
    #     denominator = 1e-5

    frac = numerator/denominator
    if isinstance(frac, pd.Series):
        return np.sqrt(frac[0])
    else:
        return np.sqrt(frac)


from scrapy.spider import Spider
from scrapy.log import logger
from scrapy import Request
from ..items import VideoItem

'''
电影 电视剧 动漫 综艺
'''

class VideoSpider(Spider):
    name = "videoSpider"
    # 电影
    start_urls = ["http://m.77kp.com/vod-type-id-1-pg-1.html"]

    def parse(self, response):
        logger.info("---------------视频页当前数据的抓取  Begin------------------------------")
        title_divs = response.xpath("//div[@class='son_nav']")
        movice_divs = response.xpath("//div[@class='list mb']")

        for i, div in enumerate(title_divs):
            title = div.xpath("//li[@class='n1']/a/text()").extract()[i]
            logger.info(title)
            movices_lis = movice_divs[i].xpath(".//ul/li")

            for item_li in movices_lis:
                name = item_li.xpath(".//a/@title").extract()
                link = "http://m.77kp.com" + item_li.xpath(".//a/@href").extract_first()
                img = item_li.xpath(".//a/img/@src").extract()
                logger.info("视频名--->{}|视频详细链接--->{}|视频图片--->{}".format(name[0], link, img))
        logger.info("---------------视频页当前数据的抓取  End------------------------------")
        logger.info("---------------类型的抓取，同时构建类型页面的request---------------")
        # 所有类型的as
        type_as = response.xpath("//ul[@class='ui-tab fn-clear']/li/a")
        typeitem = {'bigtype': '', 'smailtype': '','name':name[0]}
        for a in type_as[:len(type_as) - 1]:

            type = a.xpath("text()").extract()
            type_link = "http://m.77kp.com" + a.xpath("@href").extract_first()
            if "片" in type[0]:
                typeitem['bigtype'] = '电影'
                typeitem['smailtype'] = type[0]
            else:
                typeitem['bigtype'] = '电视剧'
                typeitem['smailtype'] = type[0]
            yield Request(response.urljoin(type_link), callback=self.parse_video, meta={'typeitem': typeitem})
            logger.info("小的类型名--->{}|链接地址---->{}".format(type[0], type_link))
        logger.info("---------------类型的抓取  End------------------------------")
        # 测试
        # yield Request(response.urljoin("http://m.77kp.com/vod-type-id-5-pg-1.html"), callback=self.parse_video,
        #               meta={'typeitem': typeitem})

    def parse_video(self, response):
        links = []
        video_as = response.xpath("//div[@class='list mb bt']/ul/li/a")
        for a in video_as:
            name = a.xpath("@title").extract()
            link = "http://m.77kp.com" + a.xpath("@href").extract_first()
            img = a.xpath(".//img/@src").extract()
            links.append(link)
            logger.info("视频名--->{}|视频详细链接--->{}|视频图片--->{}".format(name[0], link, img[0]))

        next_page = "http://m.77kp.com" + response.xpath("//div[@class='ui-pages']/a/@href")[-2].extract()
        if next_page:
            logger.info("下一页网址----->{}".format(next_page))
            yield Request(response.urljoin(next_page), callback=self.parse_video)

        for link in links:
            yield Request(response.urljoin(link), callback=self.parse_videodetail, meta=response.meta)

    def parse_videodetail(self, response):
        typeitem = response.meta['typeitem']
        title = response.xpath("//div[@class='title']/text()").extract_first()
        performer = response.xpath("//div[@class='ui-detail fn-clear']/p/text()").extract_first()
        img = response.xpath("//div[@class='ui-img']/img/@src").extract_first()
        introduction = response.xpath("//div[@class='ui-detail-info']/text()").extract()[1]

        logger.info("大类型--->{}|小类型--->{}|视频名--->{}|演员--->{}|图片--->{}|简介--->{}".format(typeitem['bigtype'],
                                                                                typeitem['smailtype'], title,
                                                                                performer, img, introduction))
        video_urls = response.xpath("//div[@class='detail-list fn-clear']/a/@href").extract()

        online_urls = []
        thunder_urls = []
        magnet_urls = []

        for url in video_urls:
            if ".html" in url:
                online_urls.append("http://m.77kp.com" + url)
            elif "thunder://" in url:
                thunder_urls.append(url)
            elif "magnet:?xt" in url:
                magnet_urls.append(url)
        logger.info("在线播放地址--->{}|迅雷下载地址--->{}|磁力链接地址---->{}".format(online_urls, thunder_urls, magnet_urls))

        videoitem = VideoItem()
        videoitem['bigtype'] = typeitem['bigtype']
        videoitem['smailtype'] = typeitem['smailtype']
        videoitem['name'] = typeitem['name']
        videoitem['pic'] = img
        videoitem['performer'] = performer
        videoitem['introduction'] = introduction
        videoitem['videohtml']=response.url
        videoitem['videourls'] = online_urls
        videoitem['thunderurls'] = thunder_urls
        videoitem['magneturls'] = magnet_urls

        yield videoitem


import os
import sys
import argparse
from tqdm import tqdm

import numpy as np
from PIL import Image

import pkgutil
from pathlib import Path
from importlib import import_module

from robosat_pink.config import load_config
from robosat_pink.tiles import tiles_from_slippy_map

def add_parser(subparser):
    parser = subparser.add_parser(
        "features",
        help="extracts simplified GeoJSON features from segmentation masks",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )

    parser.add_argument("--type", type=str, required=True, help="type of feature to extract")
    parser.add_argument("--config", type=str, required=True, help="path to configuration file")
    parser.add_argument("--path", type=str, help="path to user's extension modules dir")
    parser.add_argument("masks", type=str, help="slippy map directory with segmentation masks")
    parser.add_argument("out", type=str, help="path to GeoJSON file to store features in")

    parser.set_defaults(func=main)

def main(args):

    module_search_path = [args.path] if args.path else []
    module_search_path.append(os.path.join(Path(__file__).parent.parent, "features"))
    modules = [(path, name) for path, name, _ in pkgutil.iter_modules(module_search_path) if name != "core"]
    if args.type not in [name for _, name in modules]:
        sys.exit("Unknown type, thoses available are {}".format([name for _, name in modules]))

    config = load_config(args.config)
    labels = config["classes"]["titles"]
    if args.type not in labels:
        sys.exit("The type you asked is not consistent with yours classes in the config file provided.")
    index = labels.index(args.type)

    if args.path:
        sys.path.append(args.path)
        module = import_module(args.type)
    else:
        module = import_module("robosat_pink.features.{}".format(args.type))

    handler = getattr(module, "{}Handler".format(args.type.title()))()

    for tile, path in tqdm(list(tiles_from_slippy_map(args.masks)), ascii=True, unit="mask"):
        image = np.array(Image.open(path).convert("P"), dtype=np.uint8)
        mask = (image == index).astype(np.uint8)
        handler.apply(tile, mask)

    handler.save(args.out)


# -*- coding: utf-8 -*-
"""
Created on Sat Apr  1 10:06:23 2017

@author: samuel
"""
#加载所需模块，pandas提供excel支持，matplotlib.pyplot提供plt支持
import pandas as pd
import matplotlib.pyplot as plt
import os
#从excel中加载excel文件,目录自行修改
df = pd.read_excel(r'G:\Seafile\临时\Biodegradation of sulfur-rich oil\负离子excel\2.xlsx')
#将intensity转换为float类型
df['intensity']=df['intensity'].astype(float)
#按ppm筛选所需数据
df = df[(df.ppm>-2.0) & (df.ppm<2.0)]
#读取数据的所有化合物类，先剔除掉重复项，再将剩下的列举出来
y=df['class']
y=y.drop_duplicates()
y=y.reset_index()
m=len(y)
i=0
specie=0
#创建文件保存位置
script_dir = os.path.dirname(__file__)
results_dir = os.path.join(script_dir, '负2/')
if not os.path.isdir(results_dir):
    os.makedirs(results_dir)
#遍历上述操作找到的所有化合物类，分别绘制图谱
while i<m:
    specie=y.loc[i,'class']
    x=df[df['class']==specie]
    x['normalized']=x['intensity']/x['intensity'].sum()
    while x['normalized'].max()<100:
        x['normalized']=2*x['normalized']
    #分别绘图
    plt.figure(i)
    #设置图片格式
    font = {'family' : 'serif',
        'color'  : 'black',
        'weight' : 'normal',
        'size'   : 14,
        }
    plt.axis([0,60,0,16])
    plt.xlabel("Carbon Number",fontdict=font)
    plt.ylabel("DBE",fontdict=font)
    plt.text(1,14,s=specie,fontdict=font)
    plt.scatter(x['C'],x['DBE'],s=x['normalized'],alpha=0.8,edgecolors='white')
    sample_file_name = specie
    #保存图片
    plt.savefig(results_dir + sample_file_name,dpi=600)
    i=i+1


import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.datasets import mnist
import tensorflow as tf
gpus = tf.config.experimental.list_physical_devixe('GPU')
if gpus :
    try:
        tf.config.experimental.set_visible_devices(gpus[1],'GPU')
    except RuntimeError as e :
        print(e)

(x_train, y_train),(x_test, y_test) = mnist.load_data()

print(x_train.shape, y_train.shape)#(60000, 28, 28) (60000,)
print(x_test.shape, y_test.shape)#(10000, 28, 28) (10000,)

print(x_train[0])
print('y_train[0] : ', y_train[0])
print(x_train[0].shape)#(28, 28)

y_test1=y_test
x_train = x_train.reshape(60000,28,28,1)/255.
x_test = x_test.reshape(10000,28,28,1)/255.

#OnehotEncoding
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
encoder = OneHotEncoder()
y_train = encoder.fit_transform(y_train.reshape(-1,1)).toarray()
y_test = encoder.fit_transform(y_test.reshape(-1,1)).toarray()

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPool2D, Dense, Flatten, Dropout
from tensorflow.keras.layers import BatchNormalization, Activation
from tensorflow.keras.regularizers import l1, l2, l1_l2
model = Sequential()
model.add(Conv2D(filters=64, kernel_size=(2,2), padding='same', strides=1, input_shape=(28,28,1)))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Conv2D(32,(2,2), kernel_initializer='he_normal'))
model.add(BatchNormalization())
model.add(Activation('relu')) # relu 계열 - he_normal  sigmoid,softmax계열 - xavier

model.add(Conv2D(32,(2,2), kernel_regularizer=l1(l1=0.01)))
model.add(Dropout(0.2))

model.add(Conv2D(32,(2,2), strides=2))
model.add(MaxPool2D(pool_size=2))

model.add(Flatten())
model.add(Dense(32, activation='relu'))
model.add(Dense(10, activation='softmax'))

from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
es = EarlyStopping(monitor='val loss', patience=5)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
hist = model.fit(x_train, y_train, batch_size=128, epochs=5, validation_split=0.2, callbacks=[es])

loss, acc = model.evaluate(x_test, y_test, batch_size=200)
print('loss : ', loss)
print('accuracy : ', acc)

# loss :  0.09585337340831757
# accuracy :  0.9824000000953674


import addressify
import unittest

from parameterized import parameterized

class AddressifyTest(unittest.TestCase):

    @parameterized.expand([
        ("none argument", None, AssertionError),
        ("empty argument", "", AssertionError),
        ("single word address", "MyAddress1", AssertionError),
        ("address without number", "Street street", Exception),
        ("address without street", "18 19", Exception),
        ("address with more than 5 components", "abyss street devoid of end no. 555", AssertionError)
    ])
    def test_invalid_address(self, name, input_data, expected_result):
        with self.assertRaises(expected_result):
            addressify.parse(input_data)

    @parameterized.expand([
        ("common german address", "Winterallee 3", {
            "street": "Winterallee",
            "housenumber": "3"
        }),
        ("german address with letter on number", "Blaufeldweg 123B", {
            "street": "Blaufeldweg",
            "housenumber": "123B"
        }),
        ("german adress with multiple words", "Am Bächle 23", {
            "street": "Am Bächle",
            "housenumber": "23"
        }),
        ("german address with multiple words and number with letter already spaced", "Auf der Vogelwiese 23 b", {
            "street": "Auf der Vogelwiese",
            "housenumber": "23 b"
        }),
        ("french address", "4, rue de la revolution", {
            "street": "rue de la revolution",
            "housenumber": "4"
        }),
        ("north american address", "200 Broadway Av", {
            "street": "Broadway Av",
            "housenumber": "200"
        }),
        ("spanish address", "Calle Aduana, 29", {
            "street": "Calle Aduana",
            "housenumber": "29"
        }),
        ("spanish address name has number on it", "Calle 39 No 1540", {
            "street": "Calle 39",
            "housenumber": "No 1540"
        }),
        ("made up address starting with number using letter", "574C, obliteration street", {
            "street": "obliteration street",
            "housenumber": "574C"
        }),
        ("chinese address using number abbreviation with period", "NO. 36 BAOSHAN JIUCUN", {
            "street": "BAOSHAN JIUCUN",
            "housenumber": "NO. 36"
        }),
        ("made up address with exactly 5 components", "forsaken abyss street no. 555", {
            "street": "forsaken abyss street",
            "housenumber": "no. 555"
        })
    ])
    def test_parse(self, name, input_data, expected_result):
        result = addressify.parse(input_data)
        self.assertDictEqual(expected_result, result)

    @parameterized.expand([
        ("north american addresss", "200 doomsday street", True),
        ("spanish address", "Calle del L'via L'Viaquez 3", False),
        ("german address", "Am Bächle 23", False),
        ("chinese address", " NO. 36 BAOSHAN JIUCUN", True)
    ])
    def test_guess_if_address_starts_with_number(self, name, input_data, expected_result):
        test_input = input_data.split()
        result = addressify.does_it_starts_with_number(test_input)
        self.assertEqual(expected_result, result)

    @parameterized.expand([
        ("abbreviation with period", "nO.", True),
        ("abbreviation without period", "no", True),
        ("no abbreviation", "on", False),
        ("unclear abbreviation", "-No.", False)
    ])
    def test_is_number_abbreviation(self, name, input_data, expected_result):
        result = addressify.is_number_prefix(input_data)
        self.assertEqual(expected_result, result)

    @parameterized.expand([
        ("only digits", "555", True),
        ("digits with letter", "1548B", True),
        ("digits with comma", "1548B,", True),
        ("no digits", "true", False)
    ])
    def test_internal_has_digit(self, name, input_data, expected_result):
        result = addressify.has_digit(input_data)
        self.assertEqual(expected_result, result)


import lxml.html
import datetime
import dateutil.relativedelta
import logging
import requests
import os
import tarfile
from io import BytesIO

class WikipediaLogDownloader:

    def __init__(self, folder=""):
        self.folder = folder
        if not os.path.exists(folder):
                os.makedirs(folder)
        self.log = logging.getLogger(__name__)
        logging.getLogger("requests").setLevel(logging.WARNING)

    def download_file(self, file_url, file_date, hour, tgz=None):
        r = requests.get(file_url)
        if r.status_code != 200:
            self.log.error("Can't get file %s", file_url)
            return

        if self.folder != "":
            folder = self.folder + "/"
        else:
            folder = self.folder

        if tgz is None:
            try:
                with open(folder + str(file_date) +
                          "_" + hour + ".log", "w") as f:
                    f.write(r.text)
            except:
                self.log.error("Error for file " + file_url)
                with open(folder + "error_log.txt", "a") as ef:
                    ef.write("Error for file " + file_url)
                return
        else:
            try:
                info = tarfile.TarInfo(str(file_date) + "_" + hour + ".log")
                info.size = len(r.content)
                tgz.addfile(info, BytesIO(bytes(r.content)))
            except:
                self.log.error("Error for file " + file_url)
                with open(folder + "error_log.txt", "a") as ef:
                    ef.write("Error for file " + file_url)
                return

    def obsolete_download_month(self, current_date, start_date, end_date):
        '''Downloads all the files for the month of current_date as long as they are
        before end_date. Writes all of them in a single .tgz file.
        All dates are datetime.date objects.
        '''
        base_url = 'http://dumps.wikimedia.org/other/pagecounts-raw'

        year = str(current_date.year)
        month = "{:02d}".format(current_date.month)

        index_url = base_url + '/' + year + '/' + year + "-" + month

        dom = lxml.html.fromstring(requests.get(index_url).text)

        tgz = tarfile.open(self.folder + "/" +
                           datetime.date(int(year), int(month)) +
                           ".tgz", mode="w:bz2")
        for link in dom.xpath('//a/@href'):
            # select the url in href for all a tags
            if (link.startswith('projectcounts')):
                hour = link[-6:-4]
                day = link[-9:-7]
                file_url = index_url + '/' + link
                file_date = datetime.date(int(year), int(month), int(day))
                if start_date <= file_date < end_date:
                    self.download_file(file_url, file_date, hour, tgz=tgz)
        tgz.close()

    def download_month(self, year, month):
        '''Downloads all the files for the month of current_date as long as they are
        before end_date. Writes all of them in a single .tgz file.
        All dates are datetime.date objects.
        '''
        base_url = 'http://dumps.wikimedia.org/other/pagecounts-raw'

        start_date = datetime.date(year, month, 1)
        end_date = start_date + dateutil.relativedelta.relativedelta(months=+1)
        year = str(year)
        month = "{:02d}".format(month)

        index_url = base_url + '/' + year + '/' + year + "-" + month

        dom = lxml.html.fromstring(requests.get(index_url).text)

        tgz = tarfile.open("{}/{}-{}.tbz".format(self.folder, year, month),
                           mode="w:bz2")
        for link in dom.xpath('//a/@href'):
            # select the url in href for all a tags
            if (link.startswith('projectcounts')):
                hour = link[-6:-4]
                day = link[-9:-7]
                file_url = index_url + '/' + link
                file_date = datetime.date(int(year), int(month), int(day))
                if start_date <= file_date < end_date:
                    self.download_file(file_url, file_date, hour, tgz=tgz)
        tgz.close()

    def download(self, start_year, start_month, start_day,
                 number_of_days, folder=""):
        start_date = datetime.date(start_year, start_month, start_day)
        end_date = start_date + datetime.timedelta(days=number_of_days)

        self.folder = folder

        # Iterate over the months, as there is one index page per month
        current_date = start_date
        while current_date < end_date:
            self.download_month(current_date, start_date, end_date)
            current_date = (current_date +
                            dateutil.relativedelta.relativedelta(months=+1))


from angular_flask import app
import influxdb
from influxdb import InfluxDBClient

class ConnectInflux():
	def __init__(self):
		self.client = InfluxDBClient('influxdb-prod.juspay.in',8086,'realtime-user', 'MgqEK,nWx2bV','godel-realtime')

	def acsMetrics(self):
		query1 = "select count(distinct(session_id)) as sess_count from \
				payment_details_data where time > now() - 5m and session_id<>'null' and \
				bank<>'null' and app_name<>'null' group by \
				acs_hash"
		query2 = "select count(distinct(session_id)) from \
				payment_details_data where time > now() - 5m and session_id<>'null' and \
				bank<>'null' and app_name<>'null' and payment_status='SUCCESS' group by \
				acs_hash"
		query3 = "select count(distinct(session_id)) ,sum(potential_payment_flow_error) \
				  as pay_error,sum(user_error) as user_error ,sum(godel_exception) as exp,\
				  sum(acs_error) as acs_error from session_aggregates where time> now() - 5m \
				  and session_id<>'null' and bank<>'null' and app_name<>'null' group by acs_hash"
		result1 = self.client.query(query1)
		result2 = self.client.query(query2)
		result3 = self.client.query(query3)

		join_list = {}
		for i in result1[0]['points']:
			join_list[i[2]] = [i[1],0,0,0,0,0]  #l-4 :u'pay_error', u'user_error', u'exp', u'acs_error', u'acs_hash'

		for i in result2[0]['points']:
			if i[2] in join_list:
				join_list[i[2]][1] = i[1]
			else:
				join_list[i[2]] = [0,i[1],0,0,0,0]

		for i in result3[0]['points']:
			if i[6] in join_list:
				join_list[i[6]][2] = i[2]
				join_list[i[6]][3] = i[3]
				join_list[i[6]][4] = i[4]
				join_list[i[6]][5] = i[5]
			else:
				join_list[i[6]] = [0,0,i[2],i[3],i[4],i[5]]

		res = {'columns':['acs_hash','tot_sess_count','success_count','success_rate','potential_error', 'user_error', 'godel_exception', 'acs_error'],'rows':[]}
		map_column = {}
		for i in join_list:
			row = []
			row.append(i)
			row.append(join_list[i][0])
			row.append(join_list[i][1])
			if join_list[i][0]==0:
				row.append('not-def')
			else:
				row.append(round(float(join_list[i][1])/float(join_list[i][0])*100,2))
			row.append(join_list[i][2])
			row.append(join_list[i][3])
			row.append(join_list[i][4])
			row.append(join_list[i][5])
			res['rows'].append(row)

		return res

	def liveSessionStream(self):
		query = "select session_id, acs_hash, bank, name, numscreens, numevents, acs_error,\
				 potential_payment_flow_error,user_error,godel_exception from \
				 internal_session_aggregates where time > now() - 5m"

		res = {'columns':[],'rows':[]}
		result = self.client.query(query)
		columns=['session_id','acs_hash','bank','name' ,'numscreens','acs_error', 'potential_payment_flow_error','user_error','godel_exception']
		map_list = {}

		for i,j in enumerate(result[0]['columns']):
			map_list[j] =i

		for i in result[0]['points']:
			arr = []
			for j in columns:
				arr.append(i[map_list[j]])
			res['rows'].append(arr)

		res['columns'] = columns
		return res


import requests
import re
from requests.exceptions import RequestException
import json
from multiprocessing import Pool
import pymongo

client = pymongo.MongoClient('localhost')
db = client['maoyan']

def get_html(url):
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 \
    (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}
    try:
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        return response.text
    except RequestException:
        print('Failed to connect')

def parse_html(html):
    pattern = re.compile('
.*?board-index.*?>(\d+).*?title="(.*?)".*?data-src="(.*?)".*?star">'
    +'(.*?)
.*?releasetime">(.*?)
.*?integer">(.*?).*?fraction">(.*?)
', re.S)
    items = re.findall(pattern, html)
    for item in items:
        yield {
            'index': item[0],
            'title': item[1],
            'image': item[2],
            'actor': item[3].strip()[3:],
            'time': item[4].strip()[5:],
            'score': item[5]+item[6]
        }

def save_to_file(content):
    with open('movie100.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(content, ensure_ascii=False) + '\n')
        f.close()

def main(page):
    url = 'http://maoyan.com/board/4?offset=' + str(page)
    html = get_html(url)
    items = parse_html(html)
    n = 1
    for item in items:
        #save_to_file(item)
        if db["maoyan"].insert(item):
            print("第 %d 条保存到数据库成功" % n)
            n += 1

if __name__ == '__main__':
    #pool = Pool()
    #pool.map(main, [i*10 for i in range(10)])
    for i in range(10):
        main(i * 10)


from django.forms import Form, ModelForm
from django import forms
from todo_maker.models import Task
from django.utils.safestring import mark_safe

class TaskCreateForm(ModelForm):
    id = forms.CharField(required=False)
    task_accomplished = forms.ChoiceField(
            widget = forms.Select(attrs={'class': "form-control"}),
            choices=[(0, "Task Pending"), (1, "Task Accomplished")],
        )
    class Meta:
        model = Task
        fields = ("id", "title", "description", "task_accomplished",)
        labels = {
            "title" : mark_safe("Task Title {}".format("*")),
            "task_accomplished" : mark_safe("Task accomplished {}".format("*")),
        }
        widgets = {
            "title" : forms.TextInput(attrs={'class' : 'form-control'}),
            "description" : forms.Textarea(attrs={'class' : 'form-control', 'rows' : 2}),
        }

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)


import calendar
from datetime import date

_weekday_to_index = {
    'Monday': calendar.MONDAY,
    'Tuesday': calendar.TUESDAY,
    'Wednesday': calendar.WEDNESDAY,
    'Thursday': calendar.THURSDAY,
    'Friday': calendar.FRIDAY,
    'Saturday': calendar.SATURDAY,
    'Sunday': calendar.SUNDAY,
}

_nth_map = {
    '1st': lambda days: days[0][0],
    'first': lambda days: days[0][0],
    '2nd': lambda days: days[1][0],
    'second': lambda days: days[1][0],
    '3rd': lambda days: days[2][0],
    'third': lambda days: days[2][0],
    '4th': lambda days: days[3][0],
    'fourth': lambda days: days[3][0],
    'last': lambda days: days[-1:][0][0],
    'teenth': lambda days: list(
        filter(lambda d: d[0] >= 13 and d[0] <= 19,
                days))[0][0]
}

def nth_to_index(nth: str, days: list) -> int:
    get_index = _nth_map.get(nth)

    if get_index:
        return get_index(days)
    else:
        raise ValueError("%s is not an index in %s" %
                         (nth, sorted(_nth_map.keys())))

def meetup_day(year: int, month: int, day: str, nth: str) -> date:
    cal = calendar.Calendar()

    days = [(md, wd) for md, wd in cal.itermonthdays2(year, month)
            if _weekday_to_index[day] == wd and md != 0]

    target_day = nth_to_index(nth, days)

    return date(year, month, target_day)


# uncompyle6 version 3.7.4
# Python bytecode 3.4 (3310)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /usr/lib64/python3.4/site-packages/PyIRC/extensions/altnick.py
# Compiled at: 2015-10-08 05:15:23
# Size of source mod 2**32: 3758 bytes
"""Some alternate nick handlers.

This contains an underscore-appending handler and a number-substituting
(leetifying) handler.

"""
from logging import getLogger
from taillight.signal import SignalStop
from PyIRC.signal import event
from PyIRC.numerics import Numerics
from PyIRC.extensions import BaseExtension
_logger = getLogger(__name__)

class UnderscoreAlt(BaseExtension):
    __doc__ = 'This class attempts to append underscores to the nickname.\n\n    If :py:class:`~PyIRC.extensions.ISupport` is present, it will try until\n    the maximum nick length is reached; otherwise, it will try 5 times.\n\n    '

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.attempt_nick = self.nick
        self.attempts = 0

    @event('commands', Numerics.ERR_NICKNAMEINUSE, priority=-1000)
    @event('commands', Numerics.ERR_ERRONEOUSNICKNAME, priority=-1000)
    @event('commands', Numerics.ERR_NONICKNAMEGIVEN, priority=-1000)
    def change_nick(self, _, line):
        """Try to complete registration with a long _."""
        if self.registered:
            raise SignalStop()
        isupport = self.get_extension('ISupport')
        if not isupport:
            if self.attempts_count >= 5:
                return
        elif len(self.attempt_nick) == isupport.get('NICKLEN'):
            return
        self.attempt_nick += '_'
        self.attempts += 1
        self.send('NICK', [self.attempt_nick])
        raise SignalStop()

class NumberSubstitueAlt(BaseExtension):
    __doc__ = 'This class attempts to substitute letters for numbers and vis versa.\n\n    This extension will try until all opportunities for leetifying have been\n    exhausted.\n\n    '
    leetmap = {'A': '4',
     'a': '4',
     'B': '8',
     'E': '3',
     'e': '3',
     'G': '6',
     'g': '9',
     'I': '1',
     'i': '1',
     'O': '0',
     'o': '0',
     'S': '5',
     's': '5',
     'T': '7',
     't': '7',
     '`': '\\'}
    unleetmap = {v:k for k, v in leetmap.items()}

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.attempt_nick = self.nick
        self.index = 0

    @event('commands', Numerics.ERR_NICKNAMEINUSE, priority=-1000)
    @event('commands', Numerics.ERR_ERRONEOUSNICKNAME, priority=-1000)
    @event('commands', Numerics.ERR_NONICKNAMEGIVEN, priority=-1000)
    def change_nick(self, _, line):
        """Try to complete registration by being a 1337 h4x0r."""
        if self.registered:
            raise SignalStop()
        while self.index < len(self.attempt_nick):
            char = self.attempt_nick[self.index]
            if self.index > 0 and char in self.leetmap:
                char = self.leetmap[char]
            else:
                if char in self.unleetmap:
                    char = self.unleetmap[char]
                else:
                    self.index += 1
                    continue
            self.attempt_nick = self.attempt_nick[:self.index] + char + self.attempt_nick[self.index + 1:]
            self.send('NICK', [self.attempt_nick])
            self.index += 1
            raise SignalStop()


# !/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 31 16:23:13 2018

@author: samschott
"""
# system imports
import os.path as osp
import time
from distutils.version import LooseVersion

# external packages
from PyQt5 import QtGui, QtCore, QtWidgets, uic

# maestral modules
from maestral import __version__, __author__, __url__
from maestral.utils.appdirs import get_home_dir
from maestral.utils.notify import FILECHANGE, SYNCISSUE
from maestral.utils.autostart import AutoStart

# local imports
from .selective_sync_dialog import SelectiveSyncDialog
from .resources import (get_native_item_icon, UNLINK_DIALOG_PATH,
                        SETTINGS_WINDOW_PATH, APP_ICON_PATH, FACEHOLDER_PATH)
from .utils import (
    UserDialog,
    get_scaled_font, isDarkWindow, center_window,
    LINE_COLOR_DARK, LINE_COLOR_LIGHT,
    icon_to_pixmap, get_masked_image, MaestralBackgroundTask
)

NEW_QT = LooseVersion(QtCore.QT_VERSION_STR) >= LooseVersion('5.11')

class UnlinkDialog(QtWidgets.QDialog):

    # noinspection PyArgumentList
    def __init__(self, mdbx, restart_func, parent=None):
        super().__init__(parent=parent)
        # load user interface layout from .ui file
        uic.loadUi(UNLINK_DIALOG_PATH, self)

        self.setWindowFlags(QtCore.Qt.Sheet)
        self.setModal(True)

        self.restart_func = restart_func
        self.mdbx = mdbx

        self.buttonBox.buttons()[0].setText('Unlink')
        self.titleLabel.setFont(get_scaled_font(bold=True))
        self.infoLabel.setFont(get_scaled_font(scaling=0.9))

        icon = QtGui.QIcon(APP_ICON_PATH)
        pixmap = icon_to_pixmap(icon, self.iconLabel.width(), self.iconLabel.height())
        self.iconLabel.setPixmap(pixmap)

    def accept(self):

        self.buttonBox.setEnabled(False)
        self.progressIndicator.startAnimation()
        self.unlink_thread = MaestralBackgroundTask(self, self.mdbx.config_name, 'unlink')
        self.unlink_thread.sig_done.connect(self.restart_func)

# noinspection PyArgumentList
class SettingsWindow(QtWidgets.QWidget):
    """A widget showing all of Maestral's settings."""

    _update_interval_mapping = {0: 60*60*24, 1: 60*60*24*7, 2: 60*60*24*30, 3: 0}

    def __init__(self, parent, mdbx):
        super().__init__()
        uic.loadUi(SETTINGS_WINDOW_PATH, self)
        self.setWindowFlags(self.windowFlags() | QtCore.Qt.WindowStaysOnTopHint)
        self._parent = parent
        self.update_dark_mode()

        self.adjustSize()

        self.mdbx = mdbx
        self.selective_sync_dialog = SelectiveSyncDialog(self.mdbx, parent=self)
        self.unlink_dialog = UnlinkDialog(self.mdbx, self._parent.restart, parent=self)
        self.autostart = AutoStart(self.mdbx.config_name, gui=True)

        self.labelAccountName.setFont(get_scaled_font(1.5))
        self.labelAccountInfo.setFont(get_scaled_font(0.9))
        self.labelSpaceUsage.setFont(get_scaled_font(0.9))

        # fixes sizes of label and profile pic
        self.labelAccountName.setFixedHeight(self.labelAccountName.height())
        self._profile_pic_height = round(self.labelUserProfilePic.height() * 0.65)

        self.refresh_gui()

        # update profile pic and account info periodically
        self.update_timer = QtCore.QTimer()
        self.update_timer.timeout.connect(self.refresh_gui)
        self.update_timer.start(5000)  # every 5 sec

        # connect callbacks
        self.pushButtonUnlink.clicked.connect(self.unlink_dialog.exec_)
        self.pushButtonExcludedFolders.clicked.connect(self.selective_sync_dialog.populate_folders_list)
        self.pushButtonExcludedFolders.clicked.connect(self.selective_sync_dialog.open)
        self.checkBoxStartup.stateChanged.connect(self.on_start_on_login_clicked)
        self.checkBoxNotifications.stateChanged.connect(self.on_notifications_clicked)
        self.checkBoxAnalytics.stateChanged.connect(self.on_analytics_clicked)
        self.comboBoxUpdateInterval.currentIndexChanged.connect(
            self.on_combobox_update_interval)
        self.comboBoxDropboxPath.currentIndexChanged.connect(self.on_combobox_path)
        msg = ('Choose a location for your Dropbox. A folder named "{0}" will be ' +
               'created inside the folder you select.'.format(
                   self.mdbx.get_conf('main', 'default_dir_name')))
        self.dropbox_folder_dialog = QtWidgets.QFileDialog(self, caption=msg)
        self.dropbox_folder_dialog.setModal(True)
        self.dropbox_folder_dialog.setAcceptMode(QtWidgets.QFileDialog.AcceptOpen)
        self.dropbox_folder_dialog.setFileMode(QtWidgets.QFileDialog.Directory)
        self.dropbox_folder_dialog.setOption(QtWidgets.QFileDialog.ShowDirsOnly, True)
        self.dropbox_folder_dialog.fileSelected.connect(self.on_new_dbx_folder)
        self.dropbox_folder_dialog.rejected.connect(
                lambda: self.comboBoxDropboxPath.setCurrentIndex(0))

        center_window(self)

    @QtCore.pyqtSlot()
    def refresh_gui(self):

        # populate account info
        self.set_profile_pic_from_cache()
        self.set_account_info_from_cache()

        # populate sync section
        parent_dir = osp.split(self.mdbx.dropbox_path)[0]
        relative_path = self.rel_path(parent_dir)
        folder_icon = get_native_item_icon(parent_dir)

        self.comboBoxDropboxPath.clear()
        self.comboBoxDropboxPath.addItem(folder_icon, relative_path)
        self.comboBoxDropboxPath.insertSeparator(1)
        self.comboBoxDropboxPath.addItem(QtGui.QIcon(), 'Other...')

        # populate app section
        self.checkBoxStartup.setChecked(self.autostart.enabled)
        self.checkBoxNotifications.setChecked(self.mdbx.notification_level == FILECHANGE)
        self.checkBoxAnalytics.setChecked(self.mdbx.analytics)
        update_interval = self.mdbx.get_conf('app', 'update_notification_interval')
        closest_key = min(
            self._update_interval_mapping,
            key=lambda x: abs(self._update_interval_mapping[x] - update_interval)
        )
        self.comboBoxUpdateInterval.setCurrentIndex(closest_key)

        # populate about section
        year = time.localtime().tm_year
        self.labelVersion.setText(self.labelVersion.text().format(__version__))
        self.labelUrl.setText(self.labelUrl.text().format(__url__))
        self.labelCopyright.setText(self.labelCopyright.text().format(year, __author__))

    def set_profile_pic_from_cache(self):

        try:
            pixmap = get_masked_image(self.mdbx.account_profile_pic_path, size=self._profile_pic_height)
        except OSError:
            pixmap = get_masked_image(FACEHOLDER_PATH, size=self._profile_pic_height)

        self.labelUserProfilePic.setPixmap(pixmap)

    def set_account_info_from_cache(self):

        acc_display_name = self.mdbx.get_state('account', 'display_name')
        acc_mail = self.mdbx.get_state('account', 'email')
        acc_type = self.mdbx.get_state('account', 'type')
        acc_space_usage = self.mdbx.get_state('account', 'usage')
        acc_space_usage_type = self.mdbx.get_state('account', 'usage_type')

        if acc_space_usage_type == 'team':
            acc_space_usage += ' (Team)'

        # if the display name is longer than 230 pixels, reduce font-size
        default_font = get_scaled_font(1.5)
        if NEW_QT:
            account_display_name_length = QtGui.QFontMetrics(default_font).horizontalAdvance(acc_display_name)
        else:
            account_display_name_length = QtGui.QFontMetrics(default_font).width(acc_display_name)
        if account_display_name_length > 240:
            font = get_scaled_font(scaling=1.5*240/account_display_name_length)
            self.labelAccountName.setFont(font)
        self.labelAccountName.setText(acc_display_name)

        if acc_type != '':
            acc_type_text = ', Dropbox {0}'.format(acc_type.capitalize())
        else:
            acc_type_text = ''
        self.labelAccountInfo.setText(acc_mail + acc_type_text)
        self.labelSpaceUsage.setText(acc_space_usage)

    @QtCore.pyqtSlot(int)
    def on_combobox_path(self, idx):
        if idx == 2:
            self.dropbox_folder_dialog.open()

    @QtCore.pyqtSlot(int)
    def on_combobox_update_interval(self, idx):
        self.mdbx.set_conf('app', 'update_notification_interval',
                           self._update_interval_mapping[idx])

    @QtCore.pyqtSlot(str)
    def on_new_dbx_folder(self, new_location):

        self.comboBoxDropboxPath.setCurrentIndex(0)
        if not new_location == '':

            new_path = osp.join(new_location,
                                self.mdbx.get_conf('main', 'default_dir_name'))

            try:
                self.mdbx.move_dropbox_directory(new_path)
            except OSError:
                msg = ('Please check if you have permissions to write to the '
                       'selected location.')
                msg_box = UserDialog('Could not create directory', msg, parent=self)
                msg_box.open()  # no need to block with exec
                self.mdbx.resume_sync()
            else:
                self.comboBoxDropboxPath.setItemText(0, self.rel_path(new_location))
                self.comboBoxDropboxPath.setItemIcon(0, get_native_item_icon(new_location))

    @QtCore.pyqtSlot(int)
    def on_start_on_login_clicked(self, state):
        self.autostart.enabled = state == 2

    @QtCore.pyqtSlot(int)
    def on_notifications_clicked(self, state):
        self.mdbx.notification_level = FILECHANGE if state == 2 else SYNCISSUE

    @QtCore.pyqtSlot(int)
    def on_analytics_clicked(self, state):
        self.mdbx.analytics = state == 2

    @staticmethod
    def rel_path(path):
        """
        Returns the path relative to the users directory, or the absolute
        path if not in a user directory.
        """
        usr = osp.abspath(osp.join(get_home_dir(), osp.pardir))
        if osp.commonprefix([path, usr]) == usr:
            return osp.relpath(path, usr)
        else:
            return path

    def changeEvent(self, QEvent):

        if QEvent.type() == QtCore.QEvent.PaletteChange:
            self.update_dark_mode()

    def update_dark_mode(self):
        rgb = LINE_COLOR_DARK if isDarkWindow() else LINE_COLOR_LIGHT
        line_style = 'color: rgb({0}, {1}, {2})'.format(*rgb)

        self.line0.setStyleSheet(line_style)
        self.line1.setStyleSheet(line_style)
        self.line2.setStyleSheet(line_style)


from os import path

import pytest
import datashader as ds
import rasterio as rio

BASE_PATH = path.split(__file__)[0]
DATA_PATH = path.abspath(path.join(BASE_PATH, 'data'))
TEST_RASTER_PATH = path.join(DATA_PATH, 'world.rgb.tif')

with rio.open(TEST_RASTER_PATH) as src:
    x_range = (src.bounds.left, src.bounds.right)
    y_range = (src.bounds.bottom, src.bounds.top)
    cvs = ds.Canvas(plot_width=2,
                    plot_height=2,
                    x_range=x_range,
                    y_range=y_range)

def test_raster_aggregate_default():
    with rio.open(TEST_RASTER_PATH) as src:
        agg = cvs.raster(src)
        assert agg is not None

def test_raster_aggregate_nearest():
    with rio.open(TEST_RASTER_PATH) as src:
        agg = cvs.raster(src, resample_method='nearest')
        assert agg is not None

def test_raster_aggregate_with_overviews():
    with rio.open(TEST_RASTER_PATH) as src:
        agg = cvs.raster(src, use_overviews=True)
        assert agg is not None

def test_raster_aggregate_without_overviews():
    with rio.open(TEST_RASTER_PATH) as src:
        agg = cvs.raster(src, use_overviews=False)
        assert agg is not None

def test_out_of_bounds_return_correct_size():
    with rio.open(TEST_RASTER_PATH) as src:
        cvs = ds.Canvas(plot_width=2,
                        plot_height=2,
                        x_range=[1e10, 1e20],
                        y_range=[1e10, 1e20])
        agg = cvs.raster(src)
        assert agg.shape == (2, 2)
        assert agg is not None

def test_partial_extent_returns_correct_size():
    with rio.open(TEST_RASTER_PATH) as src:
        half_width = (src.bounds.right - src.bounds.left) / 2
        half_height = (src.bounds.top - src.bounds.bottom) / 2
        cvs = ds.Canvas(plot_width=512,
                        plot_height=256,
                        x_range=[src.bounds.left-half_width, src.bounds.left+half_width],
                        y_range=[src.bounds.bottom-half_height, src.bounds.bottom+half_height])
        agg = cvs.raster(src)
        assert agg.shape == (256, 512)
        assert agg is not None


# -*- coding: utf-8 -*-
"""
Created on Wed Sep 16 09:24:58 2015
"""

# from __future__ import division
import matplotlib
import pylab as pl
import seaborn as sbn
import h5py
import numpy as np
from collections import deque
from scipy.integrate import cumtrapz

f = h5py.File('/home/nicolas/datasave_validation_papa.h5', 'r')
list_grp = list(f.keys())
cases = []
for i, grp in enumerate(list_grp):
    cases.append(grp)

fig, axs = pl.subplots(3, 2)
axs_flat = [item for sublist in axs for item in sublist]
for n, case in enumerate(cases):
    x = f[case]['x'].value
    s = f[case]['s'].value
    y = np.linspace(np.min(s), np.max(f[case]['h'] + s), 2000)

    shape = y.shape[0], f[case]['x'].shape[0]

    h = f[case]['h'].value
    t = f[case]['t'].value
    q = f[case]['q'].value
    L = np.max(x)
    # beta = np.arctan(1/f[case].attrs['Ct'])
    # Re = f[case].attrs['Re']
    # nu = 1E-6
    # g = 9.81
    # hN = (Re * 3. * nu**2 / (g * np.sin(beta)))**(1./3.)

    # print(hN)

    ax = axs_flat[n]
    index = -1
    h_ = h[index]
    q_ = q[index]
    t_ = t[index]
    u = np.zeros(shape)
    v = np.zeros(shape)
    w = np.zeros(shape)

    flag = np.zeros(shape)

    for i, x_ in enumerate(x):
        flagsup = np.where(y <= h_[i] + s[i],
                           1, np.nan)
        flaginf = np.where(y >= s[i],
                           1, np.nan)
        for j, y_ in enumerate(y):
            u_loc = (3. * (q_[i] / h_[i]) * ((y_ - s[i]) / h_[i] -
                     (1./2) * ((y_ - s[i]) / h_[i])**2))
            u[j, i] = u_loc
        u[:, i]
        flag[:, i] = (flagsup * flaginf)
    v = -cumtrapz(np.gradient(np.nan_to_num(u))[1], y, axis=0, initial=0)
    w = cumtrapz(v, x, initial=0, axis=1)

    v *= flag
    u *= flag
    w *= flag

    magn = np.sqrt(u**2 + v**2)
    ax.plot(x, (h_ + s), color='black')
    ax.plot(x, s, color='black')

#    CS3 = ax.contour(w, 10,
#                     colors='black',
#                     extent=[x.min(), x.max(), y.min(), y.max()],
#                     aspect='auto',
#                     linewidths=.8)
    CS = ax.imshow(magn[::-1, ::1],
                   cmap='RdGy',
                   extent=[x.min(), x.max(), y.min(), y.max()],
                   aspect='auto')
    ax.legend(['t = %.0f' % t_])
    ax.set_xlim(0, x[-1])
    ax.set_ylim(-2.5, 3)
    #ax.streamplot(x, y, u, v, density=[1, 1], color='black')
pl.tight_layout
pl.show()


#!/usr/bin/env python3

import nltk
from sklearn.svm import SVC
from sklearn.model_selection import StratifiedKFold

import numpy as np
import pickle

######################################################################
CLASSIFIERS = ['maxent', 'naive_bayes', 'linear_svm', 'rbf01_svm', 'rbf02_svm']

def getROC(predictions, target_classes):
    tp, fp, tn, fn = [0] * 4
    for prediction,target_class in zip(predictions, target_classes):
        if (prediction == 1):
            if (target_class == 1):
                tp += 1
            else:
                fp += 1
        else:
            if (target_class < 1):
                tn += 1
            else:
                fn += 1

    tpr = tp/(tp + fn)
    fpr = fp/(fp + tn)
    acc = (tp + tn)/(tp + tn + fp + fn)
    return tpr, fpr, acc

def main():
    with open('nltk_featuresets.pickle', 'rb') as f:
        nltk_featuresets = pickle.load(f)
    with open('sklearn_featuresets.pickle', 'rb') as f:
        sklearn_featuresets = pickle.load(f)
    with open('target_classes.pickle', 'rb') as f:
        target_classes = pickle.load(f)

    trump_tweets_classifiers = []
    trump_tweets_classifiers.append((nltk.MaxentClassifier, "maxent"))
    trump_tweets_classifiers.append((nltk.NaiveBayesClassifier, "naive_bayes"))
    trump_tweets_classifiers.append((SVC(kernel='linear', probability=True), "linear_svm"))
    trump_tweets_classifiers.append((SVC(gamma=0.1), "rbf01_svm"))
    trump_tweets_classifiers.append((SVC(gamma=0.05), "rbf005_svm"))
    trump_tweets_classifiers.append((SVC(gamma=0.2), "rbf02_svm"))

    classifier_tpr = []
    classifier_fpr = []
    classifier_acc = []
    num_splits = 5
    cv = StratifiedKFold(n_splits=num_splits)
    for train, test in cv.split(nltk_featuresets, target_classes):
        training_set = [nltk_featuresets[i] for i in list(train)]
        testing_set = [nltk_featuresets[i] for i in list(test)]
        split_tpr = []
        split_fpr = []
        split_acc = []
        for classifier, name in trump_tweets_classifiers:
            if name == "maxent":
                trained_classifier = classifier.train(training_set, nltk.classify.MaxentClassifier.ALGORITHMS[0], max_iter=10)
            elif name == "naive_bayes":
                trained_classifier =  nltk.NaiveBayesClassifier.train(training_set)
            else:
                trained_classifier = classifier.fit(sklearn_featuresets[train], target_classes[train])

            if name == "maxent" or name == "naive_bayes":
                predictions = [trained_classifier.classify(nltk_featuresets[i][0]) for i in list(test)]
            else:
                predictions = trained_classifier.predict(sklearn_featuresets[test])

            tpr, fpr, acc = getROC(target_classes[test], predictions[:])
            split_tpr.append(tpr)
            split_fpr.append(fpr)
            split_acc.append(acc)
        classifier_tpr.append(split_tpr)
        classifier_fpr.append(split_fpr)
        classifier_acc.append(split_acc)

    for i,name in enumerate(CLASSIFIERS):
        print(name + " tpr: = {}".format(np.mean([row[i] for row in classifier_tpr])))
        print(name + " fpr: = {}".format(np.mean([row[i] for row in classifier_fpr])))
        print(name + " acc: = {}".format(np.mean([row[i] for row in classifier_acc])))

if __name__ == '__main__':
    main()


# Opmerkingen:
# Dit script kan enkel werken als je webcam nog niet in gebruik is door een andere applicatie

import cv2
import numpy as np

# Yolo model inladen en klaarzetten voor gebruik
net = cv2.dnn.readNet("weights/TinyWeightsV4.weights", "configs/TinyConfig.cfg")  # weight en configuration file ophalen
layer_names = net.getLayerNames()
output_layers = [layer_names[i[0] - 1] for i in net.getUnconnectedOutLayers()]
# Namen van de classes definiëren (volgorde is zeer belangrijk)
classes = ["Jam", "Knife", "Bread", "Choco"]
colors = np.random.uniform(0, 255, size=(len(classes), 3))

# Webcam properties instellen
cap = cv2.VideoCapture(0)  # 0 is je standaard webcam
cap.set(3, 640)  # x breedte van webcam instellen
cap.set(4, 480)  # y hoogte van webcam instellen
# cap.set(10, 150) #brightness van webcam instellen

def drawboxes(img):
    height, width, channels = img.shape

    # Detecting objects
    blob = cv2.dnn.blobFromImage(img, 0.00392, (416, 416), (0, 0, 0), True, crop=False)
    net.setInput(blob)
    outs = net.forward(output_layers)

    class_ids, confidences, boxes = [], [], []
    for out in outs:
        for detection in out:
            scores = detection[5:]
            class_id = np.argmax(scores)
            confidence = scores[class_id]
            if confidence > 0.3:
                center_x = int(detection[0] * width)
                center_y = int(detection[1] * height)
                w = int(detection[2] * width)
                h = int(detection[3] * height)

                # Rectangle coordinates
                x = int(center_x - w / 2)
                y = int(center_y - h / 2)

                boxes.append([x, y, w, h])
                confidences.append(float(confidence))
                class_ids.append(class_id)

    indexes = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
    # boxes tekenen
    for i in range(len(boxes)):
        if i in indexes:
            x, y, w, h = boxes[i]
            label = str(classes[int(class_ids[i])])
            color = colors[class_ids[i]]
            cv2.rectangle(img, (x, y), (x + w, y + h), color, 2)
            cv2.putText(img, label, (x, y + 30), cv2.FONT_HERSHEY_PLAIN, 2, color, 2)
    return img

while True:
    # Image van webcam inlezen
    success, img = cap.read()
    if not success:
        break

    # object detection uitvoeren
    img = drawboxes(img)

    # image weergeven
    cv2.imshow("frame", img)

    # zorgen dat de image zichtbaar blijft
    # druk op  de 'q' knop om de script te stoppen
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
import matplotlib.pyplot as plt

T = []
with open("T_1.csv","rb") as file:
    for line in file:
        t = float(line[:-1])
        T.append(t)

#----------------------------------------------------------

#Ahora, armamos una lista L = [l1,l2,l3,...]
#donde 'li' es una lista li = [ti1,ti2,ti3,...]
#donde 'tij' es el tiempo, en el i-esimo periodo,
#en el que llega el j-esimo foton gamma

dt = 0.003 #dwell time de 10ms
t = dt
L = []
for x in T:
    if x < t:
        try:
            l.append(x)
        except:
            l = [x]
    else:
        L.append(l)
        l = [x]
        t += dt

counts = [len(l) for l in L]
m = max(counts)+1
H = [counts.count(i) for i in range(1,m)]

plt.plot(range(1,m),H,marker='s',mfc='r',mec='k',linestyle='none')
plt.grid()
plt.show()


"""Initial migration

Revision ID: dddf4c7e2d8f
Revises:
Create Date: 2016-11-18 20:41:13.784921

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'dddf4c7e2d8f'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('urls',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('key', sa.String(length=20), nullable=True),
    sa.Column('original_url', sa.Text(), nullable=True),
    sa.Column('created', sa.DateTime(), nullable=True),
    sa.Column('last_visited', sa.DateTime(), nullable=True),
    sa.Column('visits', sa.Integer(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_urls_key'), 'urls', ['key'], unique=True)
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_urls_key'), table_name='urls')
    op.drop_table('urls')
    ### end Alembic commands ###


import torch
import torch.nn as nn
import random

from Networks import *

import sys
sys.path.insert(0, './Tasks')
from Tasks.t1 import *
from Tasks.t2 import *
from Tasks.t3 import *

# method that copies the parameters of the value network to the target value network
def hard_update(target, source):
    for target_param, param in zip(target.parameters(), source.parameters()):
        target_param.data.copy_(param.data)

# main training method which contains the training loop
def train(idx, value_network, target_value_network, optimizer, lock, counter, max_steps, epsilon_min, task,
          num_action, path, alg, discounted, beta, discount_factor, r_seed):

    DF = discount_factor
    I_update = 5
    I_target = 40000
    BETA = beta
    SAVE_EVERY = 10**5
    crit = nn.MSELoss()

    if task == 1:
        Env = task1(r_seed)
    if task == 2:
        Env = task2(r_seed)
    if task == 3:
        Env = task3(r_seed)

    f = open(path+'/log'+str(idx), 'w+')
    f.write("reward, avg\n")
    f.flush()

    t = 0
    steps = 0
    avg_reward = 0
    state = torch.Tensor([Env.state])

    # training loop
    while steps < max_steps:
        steps += 1
        t += 1

        epsilon = set_epsilon(counter.value, epsilon_min)
        # Take action a with ε-greedy policy based on Q(s, a; θ)
        action = get_egreedy_action(state, value_network, epsilon, num_action)

        # Receive new state s′ and reward r
        newObservation, reward = Env.step(action)
        next_state= torch.Tensor([newObservation])

        if (alg == 'SARSA' and steps > 1) or (alg == 'Q') or (alg == 'doubleQ'):

            if alg == 'SARSA':
                Q = computePrediction(past_state, past_action, value_network)
                Ut = computeTargets_SARSA(past_reward, state, action, DF, target_value_network, discounted,avg_reward)
            if alg == 'Q':
                Q = computePrediction(state, action, value_network)
                Ut = computeTargets_Q(reward, next_state, DF, target_value_network, discounted,avg_reward)
            if alg == 'doubleQ':
                Q = computePrediction(state, action, value_network)
                Ut = computeTargets_doubleQ(reward, next_state, DF, target_value_network, value_network, discounted,
                                            avg_reward)

            # Accumulate gradients wrt θ
            loss = crit(Q, Ut)
            loss.backward()
            avg_reward += float(BETA*(Ut-Q))

        # in this mutex lock we perform hard updates to the target value network, save the value network's
        # parameters and also log process. The lock is necessary as for the asynchronous agents to not
        # overwrite each other updates
        with lock:
            counter.value += 1

            # Update the target network θ− ← θ
            if counter.value % I_target == 0:
                hard_update(target_value_network, value_network)

            if counter.value % SAVE_EVERY == 0:
                saveModelNetwork(target_value_network, path+'/params_'+str(int(counter.value/SAVE_EVERY)))

            if counter.value % 100 == 0:
                logProgress(counter, SAVE_EVERY, 50)

            # Perform asynchronous update of θ using dθ. Clear gradients dθ ← 0.
            if t % I_update == 0:
                optimizer.step()
                optimizer.zero_grad()

            f.write("%.3f, %.2f\n" % (reward,avg_reward))
            f.flush()

        past_action = action
        past_reward = reward
        past_state = state
        state = next_state

    saveModelNetwork(value_network, path+'/params_last')
    return

# method which sets and returns epsilon parameter given the specific time-step
def set_epsilon(frames, min_e):
    e = 1 - frames/(4000000/(1-min_e))
    return max(min_e, e)

# method which performs epsilon-greedy exploration: epsilon probability of following a random action
# and 1-epsilon probability of following the best-rewarding action at this state.
def get_egreedy_action(observation, q_model, epsilon,num_actions):
    random_action= random.random()<epsilon

    if random_action:
        action= random.randint(0,num_actions-1)
    else:
        Qpreds=q_model.forward(observation).detach()
        action = Qpreds.argmax()
    return action

# method which computes and returns target update for the SARSA learning algorithm
def computeTargets_SARSA(reward, observation, action, discount_factor, target_network, discounted, avg_reward):
    q_pred = target_network.forward(observation).detach()
    if discounted:
        target = reward + discount_factor * q_pred[0][action]
    else:
        target = reward - avg_reward + q_pred[0][action]

    return target

# method which computes and returns the target update for the Q-learning algorithm
def computeTargets_Q(reward, next_observation, discount_factor, target_network, discounted, avg_reward):
    q_pred = target_network.forward(next_observation).detach()
    if discounted:
        target = reward + discount_factor * q_pred.max()
    else:
        target = reward - avg_reward + q_pred.max()
    return target

# method which computes and returns the target update for the double Q-learning algorithm
def computeTargets_doubleQ(reward, next_observation, discount_factor, target_network, value_network,
                           discounted, avg_reward):
    target_q_pred = target_network.forward(next_observation).detach()
    q_pred = value_network.forward(next_observation)[:, torch.argmax(target_q_pred)].detach()

    if discounted:
        target = reward + discount_factor * q_pred
    else:
        target = reward - avg_reward + q_pred
    return target

# method that returns the state-action value given a state, a value and a value network
def computePrediction(state, action, value_network):
    out = value_network.forward(state)
    return out[0][action]

# method saving the given model's parameters in the given directory
def saveModelNetwork(model, str_directory):
    torch.save(model.state_dict(), str_directory)

# method used to display a progress bar on the terminal
def logProgress(counter, saveIternval, barLength):
    save_iteration = int(counter.value/saveIternval)
    percent = counter.value%saveIternval * 100 / saveIternval
    makers   = '|' * int(percent/100 * barLength - 1)
    spaces  = ' ' * (barLength - len(makers))
    string = 'Parameter %d progress:\t |%s%s| %d %%' % (save_iteration, makers, spaces, percent)
    print(string.ljust(100), end='\r')


#!/usr/bin/python3

import serial, os, time

notes = ["Do", "Re", "Mi", "Fa", "Sol", "La"]

try:
    sequence = [str(notes.index(note)).encode('ascii') for note in os.sys.argv[1:]]
except ValueError:
    print("Invalid input")
    os.sys.exit(1)

try:
    connection = serial.Serial("/dev/ttyS0", baudrate=38400)

    for note in sequence:
        connection.write(note)
        answer = connection.read() # blocking call, waiting for confirmation
        time.sleep(0.3)

    print("Sequence finished.") # What gets printed last is seen on the web interface
except:
    print("Failed during communication with the device.")


from django.contrib.gis.geos import fromstr
from rest_framework import filters

class LocationFilterBackend(filters.BaseFilterBackend):
    def filter_queryset(self, request, queryset, view):
        """
        Check the query params for a location query. If present,
        filter the queryset based on the values provided, with an
        optional radius given in metres. Otherwise, just return
        the queryset.
        """
        location_query = request.query_params.get(
            'location',
            None
        )
        # Default radius of 2km
        radius_query = request.query_params.get(
            'radius',
            2000
        )
        if location_query:
            lat, lng = location_query.split(',')
            reference_location = fromstr(
                'POINT(%s %s)' % (
                    lat,
                    lng
                )
            )
            return queryset.filter(
                location__coordinates__distance_lte=(
                    reference_location,
                    radius_query
                )
            )
        return queryset


from celery.decorators import task
from testproj.clr import app as celery_app
from PumpManager.models import Song, Chart, Mix
from PumpManager.pumpdb import getSong
import logging

logging.basicConfig(filename='history.log', format='%(levelname)s:%(message)s', level=logging.DEBUG)

@celery_app.task(name = "upgr_db")
def upgr_db():
    print("START UPDATING SONG DB")
    song = "default"
    chart = "default"
    for i in range(1, 800):
        song = getSong(i)
        if song:
            try:
                print(song)
                #print(song['name'], song['author'], song['bpm'], song['type'], song['cathegory'])
                s = Song(   name=song['name'],
                            author = song['author'],
                            bpm = int(float(song['bpm'])),
                            type = song['type'],
                            cathegory = song['cathegory'],
                            )
                try:
                    s.save()
                except Exception as e:
                    logging.error(e)
                    logging.error(song)

                for mix in song['mixes']:
                    print(mix)
                    s.mix.add(Mix.objects.get(name = mix))
                    s.save()

                for chart in song['charts']:
                    c = Chart(  lvl=chart['lvl'],
                                type=chart['type'],
                                song=s,
                                )
                    try:
                        c.save()
                    except Exception as e:
                        logging.error(e)
                        logging.error(chart)

            except Exception as e:
                logging.warning("Cycle")
                logging.error(song)
                logging.error(chart)
                logging.error(e)

    print("END UPDATING SONG DB")

if __name__ == "__main__":
    upgr_db()


import os
os.chdir('E:\\Master_E\\Workspace\\bidding_learning')
import sys
#sys.path.append('../src/')
#import gym
import numpy as np
import matplotlib.pyplot as plt

from src.agent_ddpg import agent_ddpg
from src.utils import OUNoise, Memory, GaussianNoise
from src.environment_bid_market import EnvironmentBidMarket

'''
 High-Level Interface that calls learning algorithm and Energy-Market Environment
 subject to user-specified inputs

Environment Parameters

capacities:   np.array 1x(number of Agents)
costs:        np.array 1x(number of Agents)

Attention these np.arrays have to correspond to the number of Agents

Demand:       np.array 1x2
Chooses demand from arang between [min,max-1]
For fixed Demand, write: the preferred [Number, Number +1] (e.g. Demand = 99 -> [99,100])

Agents:       scalar
Number of learning agents

Rewards:      scalar
Type of Reward function. Usually 0.

Split: binary
Allow Split Bids

past_action: binary
Allow agents to learn from all agents past actions

lr_actor:    float
Learning Rate Actor

lr_critic:   float
Learning Rate Critic

Discrete:    binary
Enables Discrete Spaces (Not yet functional)
'''

POWER_CAPACITIES = [5]
PRODUCTION_COSTS = [0]
DEMAND = [15,16]
LEARNING_RATE_ACTOR = 1e-4
LEARNING_RATE_CRITIC = 1e-3
NUMBER_OF_AGENTS = 1

env = EnvironmentBidMarket(capacities = POWER_CAPACITIES, costs = PRODUCTION_COSTS, demand = DEMAND, agents = NUMBER_OF_AGENTS,
                           fringe_player = 1, rewards = 0, split = 0, past_action= 0,
                           lr_actor = LEARNING_RATE_ACTOR, lr_critic = LEARNING_RATE_CRITIC, discrete = [0, 10, 0])

agents = env.create_agents(env)
rewards = []
avg_rewards = []
# 2 different noise models

# Ohrenstein Ullenbck Noise
# This is a popular noise in machine learning.
# It starts with one distribution and then converges to another.
# Frequently, this is used to explore more in the beginning than in the end of the algorithm.
noise = OUNoise(env.action_space, mu=0.0, theta=0.15, max_sigma=0.3, min_sigma=0.3, decay_period=100000)

# Gaussian Noise
# The standard normal distributed noise with variance sigma scaled to the action spaces size
#(default: (mean = 0, sigma = 0.1) * action_space_distance)
#noise = GaussianNoise(env.action_space, mu= 0, sigma = 0.1, regulation_coef= 100, decay_rate = 0.1)

# Learning continues for a number of episodes,
# divided into batches consisting of rounds
# Each episode resets the environment, it consits of rounds
# After a number of rounds equal to the batch size, the neural networks are updated
total_episodes = 150
rounds_per_episode = 500
batch_size = 128

# Start Learning

for episode in range(total_episodes):
    state = env.reset()
    noise.reset()
    episode_reward = 0

    for step in range(rounds_per_episode):
        actions = []
        for n in range(len(agents)):
            #Neural Network Chooses Action and Adds Noise
            action_temp = agents[n].get_action(state)
            action_temp = noise.get_action(action_temp, episode)
            actions.append(action_temp[:])

        actions = np.asarray(actions)
        # Environment delivers output
        new_state, reward, done, _ = env.step(actions)

        # Add new experience to memory
        for n in range(len(agents)):
            agents[n].memory.push(state, actions[n], np.array([reward[n]]), new_state, done)

        #Update Neural Network
        if len(agents[0].memory) > batch_size:
            for n in range(len(agents)):
                agents[n].update(batch_size)

        state = new_state
        episode_reward += reward

        if done:
            sys.stdout.write("***episode: {}, reward: {}, average _reward: {} \n".format(episode, np.round(episode_reward, decimals=2), np.mean(rewards[-10:])))
            env.render()
            break

    rewards.append(episode_reward)
    avg_rewards.append(np.mean(rewards[-10:]))

plt.plot(rewards)
plt.plot(avg_rewards)
plt.plot()
plt.xlabel('Episode')
plt.ylabel('Reward')
plt.show()


from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense

classifier = Sequential()
classifier.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
classifier.add(Flatten())
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = 1, activation = 'sigmoid'))
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])

from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale = 1./255,shear_range = 0.2,zoom_range = 0.2,horizontal_flip = True)
test_datagen = ImageDataGenerator(rescale = 1./255)
print(train_datagen.__doc__)
training_set = train_datagen.flow_from_directory('train',target_size = (64, 64),batch_size = 32,class_mode = 'binary')
test_set = test_datagen.flow_from_directory('train',target_size = (64, 64),batch_size = 32,class_mode = 'binary')
classifier.fit_generator(training_set,steps_per_epoch = 8,epochs = 1000,validation_data = test_set,validation_steps = 2000)

import numpy as np
from keras.preprocessing import image
test_image = image.load_img('/Users/saranshmittal/Development/Data-Science/Cat classification/train/cat/cat.100.jpg', target_size = (64, 64))
test_image = image.img_to_array(test_image)
test_image = np.expand_dims(test_image, axis = 0)
result = classifier.predict(test_image)
if result[0][0] == 1:
    prediction = 'dog'
    print(prediction)
else:
    prediction = 'cat'
    print(prediction)


def model_to_excel(model,name="model.xlsx"):
    import openpyxl
    wb = openpyxl.Workbook()
    sheet = wb.get_active_sheet()
    sheet.title = 'Metabolites'
    sheet['A1']="Metabolite id"
    sheet['B1']="Metabolite name"
    sheet['C1']="Metabolite compartment"
    sheet['D1']="Metabolite formula"
    for n, metabolite in enumerate(model.metabolites):
        nrow=n+2
        sheet['A'+str(nrow)]=metabolite.id
        sheet['B'+str(nrow)]=metabolite.name
        sheet['C'+str(nrow)]=metabolite.compartment
        #sheet['D'+str(nrow)]=metabolite.formula.id

    sheet=wb.create_sheet(title="Reactions")
    sheet['A1']="Reaction id"
    sheet['B1']="Reaction name"
    sheet['C1']="Reaction subsystem"
    sheet['D1']="Reaction"
    for n, reaction in enumerate(model.reactions):
        nrow=n+2
        sheet['A'+str(nrow)]=reaction.id
        sheet['B'+str(nrow)]=reaction.name
        sheet['C'+str(nrow)]=reaction.subsystem
        sheet['D'+str(nrow)]=reaction.reaction

    wb.save(name)


# _*_ coding : UTF-8 _*_
# 开发团队 : 柴俊峰
# 开发人员 ：柴俊峰
# 开发时间 ：2020/12/7 23:49
# 文件名称 : 爬虫-快代理ip.py
# 开发工具 ：PyCharm

import requests
import time
from fake_useragent import UserAgent
import json
# from lxml import etree
from lxml import etree

class SpiderKuaidaili(object):
    def __init__(self):
        ua = UserAgent()
        self.base_url = "https://www.kuaidaili.com/free/inha/{}"
        self.headers = {
            'User-Agent': ua.random
        }

    #获取url列表
    def get_url_list(self,n):
        print("获取url列表")
        url_list = []
        for i in range(1,n):
            url = self.base_url.format(i)
            url_list.append(url)
        return url_list

    #请求数据
    def getPage(self,url):
        print("请求数据")
        try:
            responses = requests.get(url,headers=self.headers,timeout=5)
            if responses.status_code == 200:
                responses = responses.content.decode('utf8')
                return responses
        except:
            return False

    #解析数据
    def parseHTML(self,responses):
        print("解析数据")
        response= etree.HTML(responses)
        resips = response.xpath('//table[@class="table table-bordered table-striped"]//tr/td[1]/text()')
        resports = response.xpath('//table[@class="table table-bordered table-striped"]//tr/td[2]/text()')
        resip = list(zip(resips,resports))
        resip = [{i[0]:i[1]} for i in resip]
        print(resip)
        return resip

        # resips = []
        # resports = []
        # for resdata in resdatas:
        #     resip = resdata.xpath('./td[1]/text()')
        #     resips.append(resip)
        #     resport = resdata.xpath('./td[2]/text()')
        #     resports.append(resport)
        # # okip = list(zip(resip,resport))
        # # return okip
        # print(resips[1:])
        # print(resports[1:])
        # resip = list(zip(resips[1:],resports[1:]))
        # print(resip)

    #验证数据

    #保存数据
    def save_data(self,data):
        print("保存数据")
        with open('ip文件.json','a+') as f:
            for i in data:
                # print(json.dumps(i))
                f.write(json.dumps(i))
                f.write("\n")
        f.close()

    #统筹数据
    def run(self):
        n = int(input("请输入你想要爬取的页数："))
        url_list = self.get_url_list(n)
        for url in url_list:
            # print(f"正在爬取第{i}页")
            time.sleep(2)
            responses = self.getPage(url)
            okip = self.parseHTML(responses)
            self.save_data(okip)

if __name__ == '__main__':
    SpiderKuaidaili().run()


#!/usr/bin/python
# -*- coding: utf-8 -*-
import datetime
import re
import csv
from sortedcontainers import SortedDict

class Constants:

    @staticmethod
    def general_ranking_category():
        return "general"

    @staticmethod
    def default_ranking_category():
        return Constants.general_ranking_category()

    @staticmethod
    def ranking_categories():
        return ["economia", "gobernador", "gobierno", \
                "judicial", "legislativo", "obra.publica", \
                "pavimentacion", "presidente", "recoleccion.basura", \
                "salud", "seguridad", "servicio.agua", \
                "servicios", "transporte.publico"]

    @staticmethod
    def ranking_categories_in_general_rank():
        return ["gobernador", "legislativo", "obra.publica", "servicios", "economia", "seguridad"]

    @staticmethod
    def ranking_combinado_table_name():
        return "tbl_rank_general"

    @staticmethod
    def ranking_noticias_table_name():
        return "tbl_rank_news"

    @staticmethod
    def ranking_social_table_name():
        return "tbl_rank_tw"

    @staticmethod
    def categories_dict():
        cd = [('economia', "Economía"),
              ('gobernador', "Gobernador"),
              ('gobierno', "Gobierno"),
              ('judicial', "Jueces"),
              ('legislativo', "Diputados y Senadores"),
              ('obra.publica', "Obra pública"),
              ('pavimentacion', "Pavimentación"),
              ('presidente', "Presidente"),
              ('recoleccion.basura', "Recolección de basura"),
              ('salud', "Salud"),
              ('seguridad', "Seguridad"),
              ('servicio.agua', "Servicio de agua"),
              ('servicios', "Servicios"),
              ('transporte.publico', "Transporte público")]
        return cd

    @staticmethod
    def states_dict():
        reader = csv.reader(open("./utils/formato_estados.csv"))
        _ = reader.next()

        estados = [(row[0], row[1]) for row in reader]

        return estados

    @staticmethod
    def state_performance_categories_dict():
        cd = [('gobernador', "Gobernador"),
              ('obra.publica', "Obra pública"),
              ('pavimentacion', "Pavimentación"),
              ('recoleccion.basura', "Recolección de basura"),
              ('salud', "Salud"),
              ('seguridad', "Seguridad"),
              ('servicio.agua', "Servicio de agua"),
              ('transporte.publico', "Transporte público")]
        return cd

    @staticmethod
    def s_and_h_categories_dict():
        cd = [('presidente', 'Presidente de la república'),
              ('gobernador', 'Gobernadores'),
              ('gobierno', 'Gobierno'),
              ('legislativo', 'Diputados y Senadores'),
              ('seguridad', 'Seguridad'),
              ('servicios', 'Servicios'),
              ('economia', 'Economía')]

        return cd

    @staticmethod
    def grouping_options_dict():
        od = [('week', 'Semanal'), ('month', 'Mensual')]

        return od

class Utilities:

    @staticmethod
    def array_to_csv(array):
        result = "'" + "','".join(array) + "'"

        return  result

    @staticmethod
    def last_week_start_date():
        today = datetime.date.today()

        start_date = today - datetime.timedelta(days=today.weekday(), weeks=1)

        return start_date

    @staticmethod
    def last_week_end_date():
        end_date = Utilities.last_week_start_date() + datetime.timedelta(days=6)

        return end_date

    @staticmethod
    def last_monday_date(my_date = None):
        if my_date:
            seed_date = my_date
        else:
            seed_date = datetime.datetime.today()

        last_monday_date = seed_date - datetime.timedelta(days = seed_date.weekday())

        return last_monday_date

    @staticmethod
    def next_sunday_date(my_date = None):
        if my_date:
            seed_date = my_date
        else:
            seed_date = datetime.datetime.now()

        next_sunday_date = seed_date + datetime.timedelta(days = 6 - seed_date.weekday())

        return next_sunday_date

    @staticmethod
    def to_utf8_html_encoding(tagged_text):
        converted_text = tagged_text

        reg_exp = re.compile(r"<[a-fA-F0-9]{2}>", re.IGNORECASE)
        tags = re.findall(reg_exp, tagged_text)

        if(len(tags) > 0):
            for replaceable_tag in tags:
                html_encoded_char = str(replaceable_tag).upper().replace("<", "&#x").replace(">", ";")
                converted_text = converted_text.replace(str(replaceable_tag), html_encoded_char)

        return converted_text

    @staticmethod
    def get_category_label(category_key):
        categries_dict = dict(Constants.categories_dict())
        category_label = "General"

        for item in categries_dict.keys():
            if item == category_key:
                category_label = categries_dict[category_key]
                continue

        return category_label

    @staticmethod
    def get_state_label(state_key):
        states_dict = dict(Constants.states_dict())
        state_label = ""

        if state_key == 'pais':
            return 'País'

        for item in states_dict.keys():
            if item == state_key:
                state_label = states_dict[state_key]
                continue

        return state_label

    @staticmethod
    def get_exportpp_csv_columns_header(category):
        score_header_label = "Score"
        rank_header_label = "Rank"

        if category:
            cat_label = Utilities.get_category_label(category)
            score_header_label = " ".join([score_header_label, cat_label])
            rank_header_label = " ".join([rank_header_label, cat_label])

        headers = ("Estado", "Fecha", rank_header_label, score_header_label)

        return headers

    @staticmethod
    def get_export_states_performance_csv_column_header(category):
        score_header_label = "Score"
        rank_header_label = "Rank"

        if category:
            cat_label = Utilities.get_category_label(category)
            score_header_label = " ".join([score_header_label, cat_label])
            rank_header_label = " ".join([rank_header_label, cat_label])

        headers = ("Estado", "Inicio Periodo", "Fin Periodo", rank_header_label, score_header_label)

        return headers

    @staticmethod
    def full_state_performance_categories_dict():
        full_dict = Constants.state_performance_categories_dict()
        full_dict.insert(0, ('general', "General"))
        return full_dict

    @staticmethod
    def datepickerstring_to_date(datepickerstring):
        converted = datetime.datetime.strptime(datepickerstring, "%m/%d/%Y").date()
        return converted

    @staticmethod
    def is_number(value):
        try:
            float(value)  # for int, long and float
        except ValueError:
            return False

        return True


import numpy as np
import matplotlib.pyplot as plt
import keras
from keras import models
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.callbacks import EarlyStopping, LambdaCallback
from keras import regularizers

# Hyper parameters
batch_size = 32
num_classes = 10
epochs = 100
data_augmentation = True
num_predictions = 20
model_name = 'cnn_cifar10.h5'

# The data, split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
labels = ['Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

## Check the data
def close_on_key(event):
    plt.close()

fig = plt.figure(figsize=(12,6))
fig.canvas.mpl_connect('key_press_event', close_on_key)
ax = fig.subplots(3,6)
ax = np.reshape(ax, ax.size)
fig.suptitle("Labels: " + str(np.squeeze(y_train[:ax.size])))

for i, a in enumerate(ax):
    a.set_axis_off()
    a.imshow(np.squeeze(x_train[i]), cmap='Greys')

print("Close the window to continue!")
#plt.tight_layout()
plt.show()

# Convert class vectors to binary class matrices.
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
# 24bit colors to 32bit float colors
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
x_tensor = (1, x_train.shape[1], x_train.shape[2], x_train.shape[3])

# Load or Configure model
load_model = input("Load a pretrained model? (filename or keep blank):")
if load_model:
    # Load the model
    model = models.load_model(load_model)
    model.summary()
else:
    # Configure the model
    model = Sequential()
    model.add(
        Conv2D(32, (3, 3),
        padding='same',
        input_shape=x_train.shape[1:]
        ))
    model.add(Activation('relu'))
    model.add(Conv2D(32, (3, 3)
    ))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(64, (3, 3), padding='same'))
    model.add(Activation('relu'))
    model.add(Conv2D(64, (3, 3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(Dense(512))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes,
        kernel_regularizer=regularizers.l2(0.001)
    ))
    model.add(Activation('softmax'))

    # Compile the model with Adam optimizer
    model.compile(loss='categorical_crossentropy',
        optimizer='adam',
        metrics=['accuracy'])

    model.summary()

    def run_eval(epoch, logs):
        print("Test:")
        eval = model.evaluate(x_test, y_test)
        logs['test_loss'] = eval[0]
        logs['test_acc'] = eval[1]
        print('Test loss:', eval[0], 'Test accuracy:', eval[1])

    # Train the model
    history = model.fit(x_train, y_train,
        batch_size=batch_size,
        epochs=epochs,
        validation_split=0.1,
        shuffle=True,
        callbacks=[
            EarlyStopping(),
            LambdaCallback(
                on_epoch_end = run_eval
                )
            ]
        )
    # Save model and weights
    model.save(model_name)

    ## Plot the Losses
    fig, ax_l = plt.subplots(1)
    fig.canvas.mpl_connect('key_press_event', close_on_key)
    ax_l.plot(history.history['loss'], label='train_loss')
    ax_l.plot(history.history['val_loss'], label='val_loss')
    ax_l.plot(history.history['test_loss'], label='test_loss')
    ax_l.set_ylabel("Loss")
    ax_l.legend(loc=2)

    ## Plot the Accuracy
    ax_r = ax_l.twinx()
    ax_r.plot(history.history['acc'], '--', label='train_acc')
    ax_r.plot(history.history['val_acc'], '--', label='val_acc')
    ax_r.plot(history.history['test_acc'], '--', label='test_acc')
    ax_r.set_ylabel("Accuracy")
    ax_r.legend()

    plt.title("Learning Curve / Training Accuracy")
    plt.xlabel("Epoch")

    print("Close the window to continue!")
    plt.show()

## Plot Layer Weights
fig, ax = plt.subplots(2,2)
ax = ax.flatten()
titles = ["Conv2D_1", "Conv2D_2", "Conv2D_3", "Conv2D_4"]
ids = [0, 2, 6, 8]
fig.canvas.mpl_connect('key_press_event', close_on_key)
for i, a in enumerate(ax):
    weights, biases = model.layers[ids[i]].get_weights()
    a.set_title(titles[i])
    a.hist(weights.flatten())
    a.set_xlabel("Value Bins")
    a.set_ylabel("Occurrence")
plt.tight_layout()
plt.show()

## Qualitative examples:
class Event:
    def __init__(self, key):
        self.key = key
    pass

def sample_handler(event):
    if event.key is 'escape':
        plt.close()
    elif event.key is 'enter':
        sample = np.random.randint(x_test.shape[0])
        gt = y_test[sample].argmax()
        pred = model.predict(x_test[sample].reshape(x_tensor)).argmax()
        plt.imshow(x_test[sample].reshape(x_tensor[1:]))
        plt.title("Prediction: {}, Label: {}".format(labels[pred], labels[gt]))
        print("Prediction:", labels[pred], "Label:", labels[gt])
        plt.draw()

fig = plt.figure(figsize=(6,4))
fig.canvas.mpl_connect('key_press_event', sample_handler)
sample_handler(Event('enter'))

print("-----------------Controls-----------------")
print("Press 'Enter' for next layer.")
print("Press 'Escape' to close the plot window.")
print("Close the window to continue!")
plt.show()

## Find and Viz Mismatch
mispred = model.predict(x_test.reshape(-1, x_tensor[1], x_tensor[2], x_tensor[3])).argmax(axis=1)
mismatch =  mispred != y_test.argmax(axis=1)
print("Num of Mismatches: ", mismatch.sum())
mismatch = np.where(mismatch)[0]
print(mismatch)
mmiter = iter(mismatch)

def mismatch_handler(event):
    if event.key is 'escape':
        plt.close()
    elif event.key is 'enter':
        try:
            sample = next(mmiter)
            print(sample)
        except:
            print("No more samples!")
            plt.close()
            return
        gt = y_test[sample].argmax()
        plt.imshow(x_test[sample].reshape(x_tensor[1:]),cmap='Greys')
        plt.title("Mis-Prediction: {}, Label: {}".format(labels[mispred[sample]], labels[gt]))
        print("Mis-Prediction:", labels[mispred[sample]], "Label:", labels[gt])
        plt.draw()

fig = plt.figure(figsize=(6,4))
fig.canvas.mpl_connect('key_press_event', mismatch_handler)
mismatch_handler(Event('enter'))

print("-----------------Controls-----------------")
print("Press 'Enter' for next layer.")
print("Press 'Escape' to close the plot window.")
print("Close the window to continue!")
plt.show()

## Viz the Activation Potentials
layer_names = [model.layers[id].name for id in ids]
print(layer_names)
layer_iter = iter(layer_names)
sample = 0
name = None

def act_pot_handler(event):
    global name
    global sample
    if event.key is 'escape':
        plt.close()
        return
    elif event.key is 'enter':
        try:
            name = next(layer_iter)
        except:
            print("No more layers!")
            plt.close()
            return
    elif event.key is 'n':
        sample += 1

    layer_output = model.get_layer(name).output
    act_model = models.Model(inputs=model.input, outputs=layer_output)
    act = act_model.predict(x_test[sample].reshape(x_tensor))
    _, V, U, C = act.shape
    cells = np.ceil(np.sqrt(C)).astype(int)
    fmap = np.zeros((V*cells,U*cells))
    imn = 0
    for v in range(cells):
        for u in range(cells):
            if imn >= act.shape[-1]:
                break
            vc, uc = v*V, u*U
            fmap[vc:vc+V, uc:uc+U] = act[:,:,:,imn]
            imn += 1
    ax.set_title(name)
    ax.imshow(fmap)
    plt.draw()

fig, ax = plt.subplots(1)
fig.canvas.mpl_connect('key_press_event', act_pot_handler)
act_pot_handler(Event('enter'))

print("-----------------Controls-----------------")
print("Press 'Enter' for next layer.")
print("Press 'N' for random sample.")
print("Press 'Escape' to close the plot window.")
print("Close the window to continue!")
plt.show()


import cv2

#We need two classifiers one to detect frontal face and one to detect eye
face_clf=cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
smile=cv2.CascadeClassifier('haarcascade_smile.xml')
# we define a detect function which takes a gray scale image to processs and a normal
# image to return after processing

def facedetect(gray,frame):
    #for face we'll use the face classifier and a function which takes three
    # arguments - image,scaling factor (by how much it scales image) and min neighbours to check
    face=face_clf.detectMultiScale(gray,1.3,5)
    # face will return a tuple of 4 things- x coordinate ,y coord
    # width ,length of detected face

    #we will iterate through the faces to draw rectange over detected images

    for (x,y,w,h) in face:
        #we use rectangle function to draw rectangle on detected faces
        cv2.rectangle(frame,(x,y),(x+w,y+h),(255,0,0),2)

        # to detect smiles we will scan inside the faces and to do that we need
        # two regions of interest(roi) 1-for grascale 2- for original image

        roi_gray=gray[y:y+h,x:x+w]
        roi_color=frame[y:y+h,x:x+w]

        # just like we did for face we do for smiles
        smiles=smile.detectMultiScale(roi_gray,1.1,100)

        for (x,y,w,h) in smiles:
            cv2.rectangle(roi_color,(x,y),(x+w,y+h),(0,255,0),2)

    return frame

#Now we need to initialize webcam to record video
video_cap=cv2.VideoCapture(0)# 0 for internal webcam,1 for external webcam

while True:# We repeat infinitely (until break):
    _,frame=video_cap.read() # We get the last frame.
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # We do some colour transformations.

    canvas = facedetect(gray, frame) # We get the output of our detect function.

    cv2.imshow('Video', canvas) # We display the outputs.

    if cv2.waitKey(1) & 0xFF == ord('q'): # If we type on the keyboard:
        break

video_cap.release() # We turn the webcam off.
cv2.destroyAllWindows() # We destroy all the windows inside which the images were displayed.