# -*- coding: utf-8 -*-

"""
Copyright () 2018
All rights reserved

FILE: alg_knn.py
DATE CREATED:  @Time : 2018/2/25 10:25

DESCRIPTION: k-近邻算法
1. 该算法是用来进行分类的算法
2. 通过计算要进行分类的数据与已知类别的数据之间的距离，确定出k个近邻的数据，然后判断这k个近邻数的类别，选择出现次数最多的类别作为要进行
   分类的数据的类别
3. 常用的来计算要进行分类的数据与已知类别数据之间距离的方法：
    （1）
    （2）
4.

VERSION: : #1
CHANGE:  :
MODIFIED: : @Time : 2018/2/25 10:25
"""
import numpy as np
import operator  # 运算符模块

def create_dataset():
    """
    创建数据集合和标签
    :return:
    """
    group = np.array([[1.0, 1.1], [1.0, 1.0], [0, 0], [0, 0.1]])
    labels = ['A', 'A', 'B', 'B']
    return group, labels

def classify0(in_data, dataset, labels, k):
    """
    k-NN算法实现:
        (1)计算已知类别数据集中的点与当前点的距离
        (2)按照距离递增排序
        (3)选取距离最小的k个点
        (4)确定k个点所在类别出现的概率
        (5)返回k个点出现频率最高的类别
    return: 返回in_data所属类别
    """
    # 计算已知类别数据集中的点与当前点的距离, 采用欧式距离, 两点间的距离公式：((x0 - x1)**2 + (y0 - y1)**2)**0.5
    dataset_size = dataset.shape[0]
    diff_mat = np.tile(in_data, (dataset_size, 1)) - dataset
    sq_diff_mat = diff_mat ** 2
    sq_distance = sq_diff_mat.sum(axis=1)
    distances = sq_distance ** 0.5  # 得到已知类别中的每一个点和当前点的距离: [ 0.28284271  0.36055513  1.7691806   1.69705627]
    sorted_distaces_indices = distances.argsort()  # 得到距离的排序：[0 1 3 2]
    # 选取与当前距离最小的k个点
    class_count = {}
    for i in range(k):
        vote_label = labels[sorted_distaces_indices[i]]
        class_count[vote_label] = class_count.get(vote_label, 0) + 1
    # 排序
    sorted_class_count = sorted(class_count.items(), key=operator.itemgetter(1), reverse=True)

    return sorted_class_count[0][0]

def classify1(in_data, data_set, label, k):
    """
    计算空间两个点之间的余弦距离
    """
    n = data_set.shape[0]
    new_in_data = np.tile(in_data, (n, 1))
    a = data_set * new_in_data
    enominator_num = a.sum(axis=1)
    denominator_num = (new_in_data * new_in_data).sum(axis=1) * (data_set * data_set).sum(axis=1)
    value = enominator_num / denominator_num
    # 排序距离
    print(value)

def file2matrix(filename):
    """
    约会数据处理，将数据处理成矩阵
    """
    with open(filename, 'r') as f:
        lines = f.readlines()
        length = len(lines)
        data_matrix = np.zeros((length, 3))
        data_labels = []
        index = 0
        for line in lines:
            list_from_line = line.strip().split()
            data_matrix[index, :] = list_from_line[0:3]  # ndarry的索引：index表示第几行，第二个参数为对index进行索引
            data_labels.append(int(list_from_line[-1]))
            index += 1
        return data_matrix, data_labels

def analyse_data_by_plot(datas, label):
    """
    使用散点图进行数据分析
    """
    import matplotlib.pyplot as plt
    fig = plt.figure()  # 创建一个新的fingure
    ax = fig.add_subplot(111)
    ax.scatter(datas[:, 1], datas[:, 2], 15.0 * np.array(label), 15.0 * np.array(label))
    plt.xlabel('time play game')
    plt.ylabel('ice cream consume')
    plt.show()

def auto_norm(datas):
    """
    对数据进行归一化处理
    newvalue = (oldvalue - min) / (max - min)
    """
    min_values = datas.min(0)  # 获取datas中的每一列中的最小值, 返回该一维数组
    max_values = datas.max(0)  # 获取datas中的每一列中的最大值，返回该一维数组
    ranges = max_values - min_values
    norm_datas = np.zeros(np.shape(datas))
    m = datas.shape[0]  # 获取datas的shape，等价于np.shape(datas), m = 1000
    norm_datas = datas - np.tile(min_values, (m, 1))  # tile用来创建一个新的ndarray，重复min_value m次，每行数据重复1次
    norm_datas = norm_datas / np.tile(ranges, (m, 1))

    return norm_datas, ranges, min_values

def dating_class_test():
    """
    测试分类器的效果，得到分类器的错误率
    """
    error_rate = 0.1
    dating_data, dating_labels = file2matrix(
        '/Users/jiale/workplace/practice/machine_learning/machinelearninginaction/Ch02/datingTestSet2.txt')
    norm_mat, ranges, min_value = auto_norm(dating_data)
    m = norm_mat.shape[0]
    num_test_vec = int(m * error_rate)
    error_count = 0
    for i in range(num_test_vec):
        classifier_result = classify0(norm_mat[i, :], norm_mat[num_test_vec:m, :], dating_labels[num_test_vec:m], 4)
        print('the classifier came back with {RESULT}, the real answer is {LABELS}'.format(RESULT=classifier_result,
                                                                                LABELS=dating_labels[i]))
        if classifier_result != dating_labels[i]:
            error_count += 1
    error_rate = error_count / float(num_test_vec)
    print('the total error rate is {RATE}'.format(RATE=error_rate))

if __name__ == "__main__":
    group, labels = create_dataset()
    # 待分类数据
    in_data = np.array([1.2, 1.3])
    # 获取in_data的类别
    # classify1(in_data, group, labels, 2)
    # 使用k近邻算法进行约会网站配对效果改进
    import os

    current_folder = os.path.dirname(__file__)  # 获取当前文件的目录
    folder = os.path.join(current_folder, '../../machine_learning/machinelearninginaction/Ch02/datingTestSet2.txt')
    real_path = os.path.abspath(folder)
    a = os.path.exists(real_path)
    data, labels = file2matrix(real_path)
    # analyse_data_by_plot(data, labels)
    # 进行数据的归一化
    norm_datas, ranges, min_values = auto_norm(data)
    # 测试分类器效果, 检测分类器的错误率
    dating_class_test()


#!/usr/bin/env python
#-*-coding:utf-8

import sys,os,multiprocessing
from socket_stream_redirect import *

#redirect client output
def server1():
	mypid = os.getpid()
	conn = initListenerSocket()
	file = conn.makefile('r')
	for i in range(3):
		data = file.readline().rstrip()#read/recv client's prints
		print('server %s got: %s' % (mypid,data))
def client1():
	mypid = os.getpid()
	redirectOut()#返回一个socket，带有“写”的功能。
	for i in range(3):
		print('[client %s: %s]' %(mypid,i))#print to socket
		sys.stdout.flush()
#redirect client input
def server2():
	mypid = os.getpid()
	conn = initListenerSocket()
	for i in range(3):
		data = 'server %s :%s\n' % (mypid,i)
		conn.send(data.encode())
def client2():
	mypid = os.getpid()
	redirectIn()
	for i in range(3):
		data = input()#input from socket.
		print('client %s got: %s' % (mypid,data))

#redirect client input +output, client is socket client
def server3():
	mypid = os.getpid()
	conn = initListenerSocket()
	file = conn.makefile('r')
	for i in range(3):
		data = file.readline().rstrip()
		conn.send(('server %s got [%s]\n' % (mypid,data)).encode())

def client3():
	mypid = os.getpid()
	redirectBothAsClient()
	for i in range(3):
		print('client %s:%s ' % (mypid,i))
		data = input()
		sys.stderr.write('client %s got [%s]\n' % (mypid,data))

def server4():
#有问题
	mypid = os.getpid()
	sock = socket(AF_INET,SOCK_STREAM)
	print(host,port)
	sock.connect((host,port))
	file = sock.makefile('r')
	for i in range(3):
		sock.send(('server %s: %s\n' % (mypid,i)).encode())
		data = file.readline().rstrip()
		print('server %s got [%s]' % (mypid,data))

def client4():
	mypid = os.getpid()
	redirectBothAsClient()
	for i in range(3):
		data = input()
		print('client %s got [%s]' % (mypid,data))
		sys.stdout.flush()
def server5():
	mypid = os.getpid()
	conn = initListenerSocket()
	file = conn.makefile('r')
	for i in range(3):
		messages = 'server %s:%s\n' %(mypid,i)
		conn.send(messages.encode())
		data = file.readline().rstrip()
		print('server %s got [%s]\n' %(mypid,data))

def client5():
	mypid = os.getpid()
	redirectBothAsClient()
	for i in range(3):
		data = input()#recv
		messages = 'client %s got:[%s]\n' %(mypid,data)
		print(messages)#send
		sys.stdout.flush()

if __name__ =='__main__':
	#server1()
	multiprocessing.Process(target = server5).start()
	client5()


import argparse
import random
import sys

import numpy as np

from bouncing_objects_generator.frame import Frame
from bouncing_objects_generator.square import Square

class Cli:

    @classmethod
    def run(cls, frame_width: int, frame_height: int,
            object_width: int, object_height: int, objects_size: int,
            frames_per_window: int, window_size: int, npy_output: str):

        frame = Frame(width=frame_width, height=frame_height)
        for _ in range(objects_size):
            frame.register_object(
                Square(width=object_width, height=object_height,
                       frame_width=frame.width, frame_height=frame.height,
                       degree=random.choice(range(0, 360)),
                       speed=random.choice(range(1, 20)))
            )

        all_windows = []
        window_shape = (frames_per_window, 1, frame_height, frame_width)
        for i in range(window_size):
            sys.stderr.write(f'{i + 1}/{window_size}\r')
            sys.stderr.flush()

            window = None
            for _ in range(frames_per_window):
                image = np.asarray(frame.draw(), dtype=np.uint8).reshape(1, frame_height, frame_width)
                if window is None:
                    window = image
                else:
                    window = np.vstack((window, image))

            all_windows.append(window.reshape(window_shape[0], window_shape[1], window_shape[2], window_shape[3]))
        sys.stderr.write('\n')
        np.save(npy_output, np.concatenate(all_windows, axis=1))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--frame_width', type=int, default=128)
    parser.add_argument('--frame_height', type=int, default=128)
    parser.add_argument('--object_width', type=int, default=32)
    parser.add_argument('--object_height', type=int, default=32)
    parser.add_argument('--frames_per_window', type=int, default=20)
    parser.add_argument('--window_size', type=int, default=100)
    parser.add_argument('--objects_size', type=int, default=3)
    parser.add_argument('--npy_output', type=str, default='bouncing_objects.npy')
    args = parser.parse_args()
    Cli.run(**vars(args))

if __name__ == '__main__':
    main()


import torch
import numpy as np
import cv2
import matplotlib.pyplot as plt
from torch import nn

def compute_OKS(gt, dt, bb, area):

    sigmas = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89])/10.0
    vars = (sigmas * 2)**2
    k = len(sigmas)
    g = np.array(gt)
    xg = g[0::3]; yg = g[1::3]; vg = g[2::3]
    k1 = np.count_nonzero(vg > 0)
    x0 = bb[0] - bb[2]; x1 = bb[0] + bb[2] * 2
    y0 = bb[1] - bb[3]; y1 = bb[1] + bb[3] * 2
    d = np.array(dt)
    xd = d[0::3]; yd = d[1::3]
    if k1>0:
        # measure the per-keypoint distance if keypoints visible
        dx = xd - xg
        dy = yd - yg
    else:
        # measure minimum distance to keypoints in (x0,y0) & (x1,y1)
        z = np.zeros((k))
        dx = np.max((z, x0-xd),axis=0)+np.max((z, xd-x1),axis=0)
        dy = np.max((z, y0-yd),axis=0)+np.max((z, yd-y1),axis=0)
    e = (dx**2 + dy**2) / vars / (area.item()+np.spacing(1)) / 2
    if k1 > 0:
        e=e[vg > 0]
    OKS = np.sum(np.exp(-e)) / e.shape[0]

    return OKS

def plot_masks_on_image(im, masks):

      n_masks = masks.shape[0]
      #im = torch.transpose(im, 1, 2)
      #im = torch.transpose(im, 0, 1)
      for j in range(n_masks):

          r = cv2.resize(masks[j].numpy(), (256, 256))
          im[0, :, :] = im[0, :, :] + torch.from_numpy(r)

      return im

def get_transform(param, crop_pos, output_size, scales):
    shift_to_upper_left = np.identity(3)
    shift_to_center = np.identity(3)

    a = scales[0] * param['scale'] * np.cos(param['rot'])
    b = scales[1] * param['scale'] * np.sin(param['rot'])

    t = np.identity(3)
    t[0][0] = a
    if param['flip']:
        t[0][0] = -a

    t[0][1] = -b
    t[1][0] = b
    t[1][1] = a

    shift_to_upper_left[0][2] = -crop_pos[0] + param['tx']
    shift_to_upper_left[1][2] = -crop_pos[1] + param['ty']
    shift_to_center[0][2] = output_size / 2
    shift_to_center[1][2] = output_size / 2
    t_form = np.matmul(t, shift_to_upper_left)
    t_form = np.matmul(shift_to_center, t_form)

    return t_form

def apply_augmentation_test(example, output_size=256):
    im = cv2.imread(example, 1)
    height, width = im.shape[:2]

    crop_pos = [width / 2, height / 2]
    max_d = max(height, width)
    scales = [output_size / float(max_d), output_size / float(max_d)]

    param = {'rot': 0,
             'scale': 1,
             'flip': 0,
             'tx': 0,
             'ty': 0}

    t_form = get_transform(param, crop_pos, output_size, scales)
    im_cv = cv2.warpAffine(im, t_form[0:2, :], (output_size, output_size))
    img = cv2.cvtColor(im_cv, cv2.COLOR_BGR2RGB)
    imf = cv2.flip(img, 1)

    img = torch.from_numpy(img).float()
    img = torch.transpose(img, 1, 2)
    img = torch.transpose(img, 0, 1)
    img /= 255

    imf = torch.from_numpy(imf).float()
    imf = torch.transpose(imf, 1, 2)
    imf = torch.transpose(imf, 0, 1)
    imf /= 255

    warp = torch.from_numpy(np.linalg.inv(t_form))

    return img, imf, warp

def get_preds(prs, scales, warp):

    pool = nn.MaxPool2d(3, 1, 1).cuda()
    o = pool(prs.cuda()).data.cpu()
    maxm = torch.eq(o, prs).float()
    prs = prs * maxm
    res = 64
    prso = prs.view(res * res)
    val_k, ind = prso.topk(30, dim=0)
    xs = ind % res
    ys = (ind / res).long()
    xst, yst, sc, scores = [], [], [], []
    N = len(val_k)
    for i in range(N):
        if val_k[i] >= 0.001:
            xst.append(xs[i].item() * 4)
            yst.append(ys[i].item() * 4)
            sc.append(np.exp(scales[0][ys[i]][xs[i]]) * warp[0][0])
            scores.append(val_k[i])
    points = np.ones((3, len(sc)))
    points[0, :], points[1, :] = xst, yst
    dets = np.matmul(warp, points)

    return dets, sc, scores

def apply_augmentation_test_td(path, example, output_size=256):

    im = cv2.imread(path, 1)
    crop_pos = example['center']
    #crop_pos = [example['bbox'][0] + example['bbox'][2]/2, example['bbox'][1] + example['bbox'][3]/2]
    #x1, x2 = example['unit']['GT_bbox'][0], example['unit']['GT_bbox'][2]
    #y1, y2 = example['unit']['GT_bbox'][1], example['unit']['GT_bbox'][3]
    #crop_pos = [(x1+x2)/2, (y1+y2)/2]
    #max_d = np.maximum(abs(x1-x2), abs(y1-y2))
    max_d = example['scale']
    #max_d = np.maximum(example['bbox'][2], example['bbox'][3])
    scales = [output_size / float(max_d), output_size / float(max_d)]

    param = {'rot': 0,
             'scale': 1,
             'flip': 0,
             'tx': 0,
             'ty': 0}

    t_form = get_transform(param, crop_pos, output_size, scales)
    im_cv = cv2.warpAffine(im, t_form[0:2, :], (output_size, output_size))
    img = cv2.cvtColor(im_cv, cv2.COLOR_BGR2RGB)
    imf = cv2.flip(img, 1)

    img = torch.from_numpy(img).float()
    img = torch.transpose(img, 1, 2)
    img = torch.transpose(img, 0, 1)
    img /= 255

    imf = torch.from_numpy(imf).float()
    imf = torch.transpose(imf, 1, 2)
    imf = torch.transpose(imf, 0, 1)
    imf /= 255

    warp = torch.from_numpy(np.linalg.inv(t_form))

    return img, imf, warp

def get_preds_td(prs, mat, sr):

    pool = nn.MaxPool2d(3, 1, 1).cuda()

    xoff = sr[0:17]
    yoff = sr[17:34]

    prs2 = prs

    o = pool(prs.cuda()).data.cpu()
    maxm = torch.eq(o, prs).float()
    prs = prs * maxm
    res = 64
    prso = prs.view(17, res * res)
    val_k, ind = prso.topk(1, dim=1)
    xs = ind % res
    ys = (ind / res).long()

    keypoints = []
    score = 0
    points = torch.zeros(17, 2)
    c = 0

    for j in range(17):

        x, y = xs[j][0], ys[j][0]
        dx = xoff[j][int(y)][int(x)]
        dy = yoff[j][int(y)][int(x)]
        points[j][0] = (x * 4) + dx.item()
        points[j][1] = (y * 4) + dy.item()

        score += val_k[j][0]
        c += 1

    score /= c

    for j in range(17):

        point = torch.ones(3, 1)
        #if points[j][0] > 0 and points[j][1] > 0:
        point[0][0] = points[j][0]
        point[1][0] = points[j][1]
        #else:
        #    point[0][0] = xm
        #    point[1][0] = ym

        keypoint = np.matmul(mat, point)
        keypoints.append(float(keypoint[0].item()))
        keypoints.append(float(keypoint[1].item()))
        #keypoints.append(int(point[0][0]))
        #keypoints.append(int(point[1][0]))
        keypoints.append(1)

    return keypoints, score.item()


"""Handling settings configured within CliEN/config.json
"""
import sys

if __name__ == "__main__":
    sys.exit()

import os
import json

# i define this here instead of constants to avoid a circular dependency issue.
_CLIEN_HOME = os.path.dirname(os.path.abspath(__file__))
_CONFIG = os.path.join(_CLIEN_HOME, "config.json")

config = None

def load_config():
    """Sanity checks the user settings.
    """
    global config

    _config = json.load(open(_CONFIG, "rb"))

    if os.path.isdir(_config["PY4J_JAR_DIR_PATH"]) is False:
        raise Exception("\n\t\tERROR: PY4J_DIR_PATH directory is invalid\n")
    if os.path.isfile(_config["WORD2VEC_BIN_PATH"]) is False:
        raise Exception("\n\t\tERROR: WORD2VEC_BIN_PATH is not a file\n")
    if os.path.isfile(_config["WORD_FREQ"]) is False:
        raise Exception("\n\t\tERROR: WORD_FREQ is not a faile\n")

    config = _config

load_config()


#!/usr/bin/env python
# coding: utf-8

import os

import numpy as np
import pandas as pd
import scipy.sparse as sparse
from numpy.linalg import eigvals

work_root = os.environ['WORK']

# ESN code
def generate_reservoir(size, radius, degree, random_state):
    sparsity = degree / float(size)

    A = sparse.rand(size, size, density=sparsity, random_state=random_state).todense()
    vals = eigvals(A)
    e = np.max(np.abs(vals))

    # A = sparse.rand(size, size, density=sparsity, random_state=random_state)
    # vals = eigs(A, k=1, which='LM', return_eigenvectors=False)
    # e = np.abs(vals[0])
    # A = A.todense()

    A = (A / e) * radius
    return A

def reservoir_layer(A, Win, input, n):
    states = np.zeros((n, input.shape[1]))
    for i in range(input.shape[1] - 1):
        states[:, i + 1] = np.tanh(np.dot(A, states[:, i]) + np.dot(Win, input[:, i]))
    return states

def train(beta, states, data, n, lsp):
    idenmat = beta * sparse.identity(n)
    states2 = states.copy()
    for j in range(2, np.shape(states2)[0] - 2):
        if np.mod(j, 2) == 0:
            states2[j, :] = states[j - 1, :] * states[j - 2, :]
    U = np.dot(states2, states2.transpose()) + idenmat
    Uinv = np.linalg.inv(U)
    Wout = np.dot(Uinv, np.dot(states2, data[lsp:data.shape[0] - lsp, :].transpose()))
    return Wout.transpose()

class ESN:
    def __init__(self, radius=0.1, degree=3, sigma=0.5, approx_res_size=5000, beta=0.0001, random_state=360, lsp=0):
        self._radius = radius
        self._degree = degree
        self._sigma = sigma
        self._approx_res_size = approx_res_size
        self._beta = beta
        self._random_state = random_state
        self._lsp = lsp

        self._fn = None
        self._n = None
        self._A = None
        self._Win = None
        self._Wout = None

        self.x = None

    def fit(self, data):
        self._fn = data.shape[0]
        self._n = int(np.floor(self._approx_res_size / self._fn) * self._fn)
        self._A = generate_reservoir(self._n, self._radius, self._degree, self._random_state)

        q = int(self._n / self._fn)
        self._Win = np.zeros((self._n, self._fn))
        for i in range(self._fn):  # init input layer
            np.random.seed(seed=i)
            self._Win[i * q: (i + 1) * q, i] = self._sigma * (-1 + 2 * np.random.rand(1, q)[0])

        states = reservoir_layer(self._A, self._Win, data, self._n)
        self._Wout = train(self._beta, states, data, self._n, self._lsp)
        self.x = states[:, -1]
        return self

    def predict(self, predict_length):

        output = np.zeros((self._fn, predict_length))
        out = self.predict_next()
        output[:, 0] = out
        for i in range(1, predict_length):
            out = self.predict_next(out)
            output[:, i] = out
        return output

    def predict_next(self, u=None):
        if u is not None:
            x1 = np.tanh(np.dot(self._A, self.x) + np.dot(self._Win, u))
            self.x = np.squeeze(np.asarray(x1))
        x_aug = self.x.copy()
        for j in range(2, np.shape(x_aug)[0] - 2):
            if np.mod(j, 2) == 0:
                x_aug[j] = self.x[j - 1] * self.x[j - 2]
        out = np.squeeze(np.asarray(np.dot(self._Wout, x_aug)), axis=0)
        return out

def split_modulo(start, stop, array_len):
    if stop <= start:
        stop += array_len
    return np.arange(start, stop) % array_len

def load_data(train_length):
    pd_data = pd.read_csv(work_root + '/data/3tier_lorenz_v3.csv', header=None).T
    print(pd_data.shape)
    return np.array(pd_data)[:, :train_length]

def main():
    Q = 8
    g = 8
    q = int(Q / g)
    lsp = 3
    predict_length = 10000
    train_length = 500000
    approx_res_size = 5000

    data = load_data(train_length)

    splits = list(map(lambda i: data[split_modulo(i * q - lsp, (i + 1) * q + lsp, Q), :], range(g)))

    fitted_models = list(map(lambda x: ESN(lsp=lsp, approx_res_size=approx_res_size).fit(x), splits))

    output_parts = list(map(lambda model: model.predict_next(), fitted_models))

    output = np.zeros((Q, predict_length))
    output[:, 0] = np.concatenate(output_parts)

    input_parts = np.empty(g, dtype=object)
    for j in range(predict_length):
        output_parts = list(map(lambda model, input_part: model.predict_next(input_part), fitted_models, input_parts))
        output[:, j] = np.concatenate(output_parts)
        input_parts = list(map(lambda i: output[split_modulo(i * q - lsp, (i + 1) * q + lsp, Q), j], range(g)))

    print(output.shape)
    np.savetxt(work_root + '/data/Sequential_Expansion_2step_back_' + str(g) + '.txt', output)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-

import requests
import json
import jwt
import redis

from pythinkutils.common.log import g_logger
from pythinkutils.config.Config import g_config
from pythinkutils.common.object2json import *
from pythinkutils.redis.ThinkRedis import ThinkRedis

class TokenUtils:

    g_szHost = g_config.get("auth", "host")
    JWT_SALT = ""

    @classmethod
    def auth_token(cls, szAppId, szSecret):
        try:
            szUrl = "{}{}".format(TokenUtils.g_szHost, "/ruoyi-api/auth/token")
            resp = requests.post(szUrl, data={"appid": szAppId, "secret": szSecret})

            if 200 != resp.status_code:
                return None

            dictRet = json.loads(resp.text)
            if 200 != dictRet["code"]:
                return None

            return dictRet["token"]
        except Exception as ex:
            g_logger.error(ex)
            return None

    @classmethod
    def get_info(cls, szToken):
        try:
            szUrl = "{}{}".format(TokenUtils.g_szHost, "/ruoyi-api/getInfo")
            dictHeader = {
                "Authorization": "Bearer {}".format(szToken)
            }

            resp = requests.get(szUrl, headers = dictHeader)
            if 200 != resp.status_code:
                return None

            dictRet = json.loads(resp.text)
            if 200 != dictRet["code"]:
                return None

            return dictRet
        except Exception as ex:
            g_logger.error(ex)
            return None

    @classmethod
    def parse_token(cls, szToken):
        try:
            jwt_options = {
                'verify_signature': False,
                'verify_exp': True,
                'verify_nbf': False,
                'verify_iat': True,
                'verify_aud': False
            }

            dictToken = jwt.decode(szToken, TokenUtils.JWT_SALT, algorithms=["HS512"], options=jwt_options)
            return dictToken
        except Exception as ex:
            g_logger.error(ex)
            return None

    @classmethod
    def expire_time(cls, szToken):
        try:
            dictToken = cls.parse_token(szToken)
            if dictToken is None:
                return 0

            r = redis.StrictRedis(connection_pool=ThinkRedis.get_conn_pool_ex())

            szKey = "login_tokens:{}".format(dictToken["login_user_key"])
            return r.ttl(szKey)

        except Exception as ex:
            g_logger.error(ex)
            return 0

# szToken = TokenUtils.auth_token("1234", "5678")
# g_logger.info(szToken)
#
# dictRet = TokenUtils.get_info(szToken)
# g_logger.info(obj2json(dictRet))
#
# g_logger.info(obj2json(TokenUtils.parse_token(szToken)))
# g_logger.info(TokenUtils.expire_time(szToken))


from django.shortcuts import render, redirect
from django.contrib import auth
from MedApp.models import *
from datetime import datetime

# Create your views here.

TIME_FORMAT = '%Y-%m-%d'
DAYS = {
        'Monday' : 'Понедельник',
        'Tuesday' : 'Вторник',
        'Wednesday' : 'Среда',
        'Thursday' : 'Четверг',
        'Friday' : 'Пятница',
        'Saturday' : 'Суббота',
        'Sunday' : 'Воскресенье',
        }

def journal(request):
    if auth.get_user(request).is_active:
        out_dict = {}
        today = datetime.today().date()
        out_dict['journalapp_active'] = True
        out_dict['User'] = auth.get_user(request)
        out_dict['Clients'] = Client.objects.filter(user_id = auth.get_user(request).id)
        out_dict['ServicesForClients'] = UslugaDlaClienta.objects.filter(user_id = auth.get_user(request).id)
        out_dict['Services'] = Usluga.objects.all()
        out_dict['Today'] = datetime.today().date()
        dates = []
        days_ru = []
        for date in UslugaDlaClienta.objects.filter(user_id = auth.get_user(request).id).order_by('date'):
            if date.date not in dates:
                dates.append(date.date)
        out_dict['Dates'] = dates
        for day in dates:
            if day >= today:
                day_eng = day.strftime('%A')
                day_str = DAYS.get(day_eng)
                days_ru.append(day_str)
        out_dict['DaysRu'] = days_ru
        return render(request, 'journal/journal.html', out_dict)
    else:
        return redirect('/')

def calculate(request, date):
    return redirect('/calculate/')


#!/usr/bin/env python
import os

from setuptools import find_packages, setup

curdir = os.path.dirname(os.path.abspath(__file__))

setup(
    name='drf-nested-routers',
    description='Nested resources for the Django Rest Framework',
    long_description=open('README.md').read(),
    license='Apache',
    version='0.11.0',
    author='Alan Justino et al.',
    author_email='alan.justino@yahoo.com.br',
    url='https://github.com/alanjds/drf-nested-routers',
    install_requires=['djangorestframework>=2.4'],
    setup_requires=['setuptools'],
    packages=find_packages(curdir),
)


import sys

sys.path.append('G:/anconada/envs/py36/lib/site-packages')
from prettytable import PrettyTable
import re
import jieba
import os
import copy
from zhon.hanzi import punctuation
from scipy.misc import imread
from wordcloud import WordCloud
from wordcloud import ImageColorGenerator
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.model_selection import train_test_split
import numpy as np
import random
from prettytable import PrettyTable
import gc
import time
from gensim.models import word2vec
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
# from guppy import hpy
import json
import psutil
from gensim import corpora
from scipy.sparse import coo_matrix
from functools import partial
from scipy.special import gammaln, psi
import scipy

''' 在写论文的时候可以使用两组数据查看模型的效果'''

class ldamodel_sent():
    def __init__(self, topic_num, sentiment_num, alpha, beta, gamma,corpus,
                 batch_size, iteration,sentiment_dict):
        self.T = topic_num
        self.D = None
        self.S = sentiment_num
        self.V = None
        self.corpus=corpus
        self.wordOccurenceMatrix=None

        self.sentiment_dict = sentiment_dict  # 这是一个字典，key为word,value为list

        # 超参数设置
        self.alpha = alpha if alpha else 0.01
        self.beta = beta if beta else 0.01
        self.gamma = gamma if gamma else 0.01
        self.alpha_lz = None
        self.alphasum_l = None
        self.z_count=None
        self.l_count=None

        self.beta_lzw = None
        self.betasum_lz = None
        self.add_lw =None

        self.gamma_dl =None
        self.gammasum_d = None

        self.iteration = iteration if iteration else 1000
        self.batch_size=batch_size
        self.word2id = None
        self.id2word = None

        #
        self.coherence=None
        self.word_sentiment_vocabulary = {}
        self.sentiment_word_list = {}

        # 设置参数
        self.doc_sel_topic_count = None
        self.topic_sel_word_count = None
        self.doc_count = None
        self.topic_count = None
        self.sentiment_count = None
        self.topic_sentiment_count = None
        self.sentiment_topic_count=None
        self.sentiment_topic_word_count=None
        self.doc_sentiment_count = None
        self.sentiment_word_count=None
        self.topic_word_count=None
        self.word_sentiment_vocabulary=None

        self.doc_sel_topic = None
        self.topic_sel_word = None
        self.sentiment_topic_word=None
        self.doc_sel = None

        self.z = None
        self.l = None
        self.all_loglikelihood=[]
        self.all_perplexity=[]

    def createdictionary(self):
        self.word2id={}
        self.V=0
        self.D=0
        wordnum=0
        for i ,doc in enumerate(self.corpus):
            for j ,word in enumerate(doc):
                if word not in self.word2id.keys():
                    if word != '':
                        wordnum+=1
                        self.word2id[word]=len(self.word2id)

                    else:
                        continue

        self.V=wordnum
        self.D=len(self.corpus)
        self.id2word=dict(zip(self.word2id.values(),self.word2id.keys()))
        print('语料库中总共有{0} 个token'.format(self.V))
        print('word2id的长度：',len(self.word2id))
        print('id2word的长度',len(self.id2word))

    def sampleFromDirichlet(self,gamma):
        return np.random.dirichlet(gamma)

    def initial(self):
        self.alpha_lz = np.full((self.S, self.T), fill_value=self.alpha)
        self.alphasum_l = np.full((self.T,), fill_value=self.alpha * self.T)

        self.beta_lzw = np.full((self.S, self.T, self.V), fill_value=self.beta)
        self.betasum_lz = np.zeros((self.S, self.T), dtype=np.float)
        self.add_lw = np.ones((self.S, self.V), dtype=np.float)

        self.wordOccurenceMatrix=np.zeros((self.D,self.V))
        for i,doc in enumerate(self.corpus):
            for j,word in enumerate(doc):
                word_id=self.word2id[word]
                self.wordOccurenceMatrix[i,word_id] +=1

        for l in range(self.S):
            for z in range(self.T):
                for r in range(self.V):
                    self.beta_lzw[l][z][r] *= self.add_lw[l][r]
                    self.betasum_lz[l][z] += self.beta_lzw[l][z][r]

        self.gamma_dl = np.full((self.D, self.T), fill_value=0.0)
        self.gammasum_d = np.full(shape=(self.D), fill_value=0.0)

        for d in range(self.D):
            # self.gamma_dl[d][1] = 1.8
            self.gamma_dl[d][1] = self.gamma
            self.gamma_dl[d][2] = self.gamma
        for d in range(self.D):
            for l in range(self.S):
                self.gammasum_d[d] += self.gamma_dl[d][l]
        self.doc_sel_topic_count = np.zeros([self.D, self.S, self.T])
        print('the shape of doc_Sel_topic_count：',self.doc_sel_topic_count.shape)
        self.topic_sel_word_count = np.zeros([self.T, self.S, self.V])
        self.doc_count = np.zeros(self.D)
        self.sentiment_count = np.zeros(self.S)  ##这个不是必须的
        self.topic_count = np.zeros(self.T)  ##这个不是必须的
        self.topic_sentiment_count = coo_matrix((self.T, self.S)).toarray()
        self.doc_sentiment_count = coo_matrix((self.D, self.S)).toarray()
        self.sentiment_word_count=coo_matrix((self.S,self.V)).toarray()
        self.topic_word_count=coo_matrix((self.T,self.V)).toarray()

        self.sentiment_topic_count = coo_matrix((self.S, self.T)).toarray()
        self.sentiment_topic_word_count =np.zeros([self.S, self.T ,self.V])

        self.doc_sel_topic = np.ndarray((self.D, self.S, self.T))
        self.topic_sel_word = np.ndarray((self.T, self.S, self.V))
        self.doc_sel = np.ndarray((self.D, self.S))
        self.sentiment_topic_word=np.ndarray((self.S, self.T,self.V))

        self.z = coo_matrix((self.D, self.V), dtype=np.int8).toarray() # 存放每个文档中每一个词的主题
        self.l = coo_matrix((self.D, self.V), dtype=np.int8).toarray()  # 存放每个文档中每个词的情感极性
        self.z_count={}
        self.l_count={}
        print('开始赋予sentiment')
        gamma=[self.gamma for _ in range(self.S)]
        alpha=[self.alpha for _ in range(self.T)]
        for i, doc in enumerate(self.corpus):
            sentimentDistribution = self.sampleFromDirichlet(gamma)
            topicDistribution = np.zeros(( self.S,self.T))
            for s in range(self.S):
                topicDistribution[s, :] = self.sampleFromDirichlet(alpha)
            for j, word in enumerate(doc):
                word_id = int(self.word2id[word])
                sample_sentiment = self.sampleFromCategorical(sentimentDistribution)
                topic = self.sampleFromCategorical(topicDistribution[s,:])
                #topic = int(random.randint(0, self.T - 1))
                prior_sentiment=self.sentiment_dict[word]
                sentiment=self.sentiment_dict[word] # 需要注意的是这个地方随着查看的情感的不同需要一直改变

                #final_sentiment=random.sample([prior_sentiment,sentiment],1)[0]
                final_sentiment=prior_sentiment
                self.doc_sel_topic_count[i,final_sentiment,topic] += 1
                #print(sentiment)
                self.topic_sel_word_count[topic,final_sentiment, word_id] += 1

                self.doc_count[i] += 1
                self.sentiment_count[final_sentiment] += 1
                self.sentiment_topic_count[final_sentiment,topic]+=1
                self.sentiment_topic_word_count[final_sentiment,topic,word_id]+=1

                self.topic_count[topic] += 1
                self.topic_sentiment_count[topic, final_sentiment] += 1
                self.doc_sentiment_count[i, final_sentiment] += 1
                self.sentiment_word_count[final_sentiment,word_id]+=1
                self.topic_word_count[topic,word_id]+=1

                self.z_count[(i,j)]=topic
                self.l_count[(i,j)]=final_sentiment

    def gibbssampling(self):
        for iter in range(self.iteration):
            # 采用小批量进行训练
            for i, doc in enumerate(self.corpus):

                for j, word in enumerate(doc):
                    try:
                        word_id = int(self.word2id[word])
                        topic = int(self.z_count[(i, j)])
                        prior_sentiment=self.sentiment_dict[word]
                        sentiment=self.l_count[(i,j)]

                        if  self.doc_sel_topic_count[i, sentiment, topic]<=0:
                            self.doc_sel_topic_count[i, sentiment, topic]=0
                        else:
                            self.doc_sel_topic_count[i, sentiment, topic] -= 1

                        if self.topic_sel_word_count[topic,sentiment, word_id]<=0:
                            self.topic_sel_word_count[topic,sentiment, word_id]=0
                        else:
                            self.topic_sel_word_count[topic,sentiment, word_id] -= 1

                        if self.topic_sentiment_count[topic, sentiment]<=0:
                            self.topic_sentiment_count[topic, sentiment]=0
                        else:
                            self.topic_sentiment_count[topic, sentiment] -= 1

                        if self.doc_sentiment_count[i, sentiment]<=0:
                            self.doc_sentiment_count[i, sentiment] =0
                        else:
                            self.doc_sentiment_count[i, sentiment]-=1

                        if self.topic_count[topic]<=0:
                            self.topic_count[topic]=0
                        else:
                            self.topic_count[topic] -= 1

                        if self.sentiment_count[sentiment]<=0:
                            self.sentiment_count[sentiment]=0
                        else:
                            self.sentiment_count[sentiment] -=1

                        if self.topic_word_count[topic,word_id]<=0:
                            self.topic_word_count[topic,word_id]=0
                        else:
                            self.topic_word_count[topic,word_id]-=1

                        if self.sentiment_word_count[sentiment,word_id]<=0:
                            self.sentiment_word_count[sentiment,word_id]=0
                        else:
                            self.sentiment_word_count[sentiment,word_id]-=1

                        if self.sentiment_topic_word_count[sentiment,topic,word_id]<=0:
                            self.sentiment_topic_word_count[sentiment,topic,word_id]=0
                        else:
                            self.sentiment_topic_word_count[sentiment,topic,word_id]-=1

                        if self.sentiment_topic_count[sentiment,topic]<=0:
                            self.sentiment_topic_count[sentiment,topic]=0
                        else:
                            self.sentiment_topic_count[sentiment,topic]-=1

                        #print('resampling')
                        n_dst=self.doc_sel_topic_count[i,:,:]
                        n_tsw=self.topic_sel_word_count[:,:,word_id]
                        n_ds=self.doc_sentiment_count[i,:]
                        n_ts=self.topic_sentiment_count
                        n_d=self.doc_count[i]

                        new_topic, new_sentiment = self.resampling(n_dst,n_tsw,n_ds,n_ts,n_d)

##                        ind = self.sampleFromCategorical(probabilities_ts.flatten())
##                        new_topic, new_sentiment = np.unravel_index(ind, probabilities_ts.shape)

                        self.z_count[(i,j)]=new_topic
                        self.l_count[(i,j)]=new_sentiment

                        self.doc_sel_topic_count[i, new_sentiment, new_topic] += 1
                        self.topic_sel_word_count[ new_topic,new_sentiment , word_id] += 1
                        self.topic_sentiment_count[new_topic, new_sentiment] += 1
                        self.doc_sentiment_count[i, new_sentiment] += 1
                        self.topic_count[new_topic] += 1
                        self.sentiment_count[new_sentiment] += 1
                        self.topic_word_count[new_topic,word_id]+=1
                        self.sentiment_word_count[new_sentiment,word_id]+=1
                        self.sentiment_topic_count[new_sentiment,new_topic]+=1
                        self.sentiment_topic_word_count[new_sentiment,new_topic,word_id ]+=1

                    except Exception as result:
                        print('result:',result)

            if (iter+1)%10 == 0:
                print('开始第{0}次迭代'.format(iter+1))
        self.updateparam()
        print('训练过程结束')

    def updateparam(self):
        for i in range(self.D):
            for j in range(self.S):
                self.doc_sel[i, j] = (self.doc_sentiment_count[i, j] + 0.01)/(self.doc_count[i] +
                                                                                self.S * 0.01)

        for i in range(self.T):
            for j in range(self.S):
                for k in range(self.V):
                    self.topic_sel_word[i, j, k] = (self.topic_sel_word_count[i, j, k] + self.beta_lzw[j][i][k])/\
                                                   (self.topic_sentiment_count[i, j] + self.betasum_lz[j][i])

        for i in range(self.S):
            for j in range(self.T):
                for k in range(self.V):
                    self.sentiment_topic_word[i, j, k] = (self.sentiment_topic_word_count[i, j, k] + self.beta_lzw[i][j][k])/\
                                                   (self.sentiment_topic_count[i, j] + self.betasum_lz[i][j])

        for i in range(self.D):
            for j in range(self.S):
                for k in range(self.T):
                    self.doc_sel_topic[i, j, k] = (self.doc_sel_topic_count[i, j, k] + self.alpha_lz[j][k])/\
                                                  (self.doc_sentiment_count[i, j] + self.alphasum_l[j] )
        print('参数更新完成******************* \n')
        return

##
    def sampleFromCategorical(self,theta):
        theta = theta/np.sum(theta)
        return np.random.multinomial(1, theta).argmax()

    def resampling(self,n_dst,n_tsw,n_ds,n_ts,n_d ):
        pk = np.zeros([self.T, self.S])
##        for t in range(self.T):
##            for s in range(self.S):
##                pk[t,s]= float((self.doc_sel_topic_count[doc_id,s,t] + self.alpha_lz[s,t]) *\
##                          (self.topic_sel_word_count[ t,s,word_id] + self.beta ) * \
##                          (self.doc_sentiment_count[doc_id, s]+ self.gamma) / \
##                          (self.doc_sentiment_count[doc_id, s] + self.alphasum_l[s]) *\
##                          (self.topic_sentiment_count[t,s]+ self.betasum_lz[s,t]) * \
##                          (self.doc_count[doc_id] + self.gammasum_d[doc_id]))

                #pk[t, s] += pk[t, s - 1]
        for t in range(self.T):
            for s in range(self.S):
                pk[t,s]= float((n_dst[s,t] + self.alpha) *\
                          (n_tsw[t,s] + self.beta ) * \
                          (n_ds[s]+ self.gamma) / \
                          ( n_ds[s] + self.alpha * self.T) *\
                          (n_ts[t,s]+ self.beta * self.V) * \
                          (n_d + self.gamma * self.S))

        u = random.random()
        flag=0
        for j in range(self.T):
            for k in range(self.S):
                if pk[j, k] >= u:
                    flag=1
                    return j,k
        if flag==0:
            se = random.randint(0, self.S - 1)
            to = random.randint(0, self.T - 1)
            return to, se
##        pk /= np.sum(pk)
##        return pk

##    def log_multi_beta(self,alpha, K=None):
##        if K is None:
##        # alpha is assumed to be a vector
##            return np.sum(scipy.special.gammaln(alpha)) - scipy.special.gammaln(np.sum(alpha))
##        else:
##        # alpha is assumed to be a scalar
##            return K * scipy.special.gammaln(alpha) - scipy.special.gammaln(K*alpha)
##
##    def loglikelihood(self):
##        n_docs=len(self.corpus)
##        lik = 0
##
##        for z in range(self.T):
##            for s in range(self.S):
##                lik += self.log_multi_beta(self.topic_sel_word_count[z, s,:]+self.beta)
##
##        lik -= self.T * self.S * self.log_multi_beta(self.beta, self.V)
##
##        for m in range(n_docs):
##            for z in range(self.S):
##                lik += self.log_multi_beta(self.doc_sel_topic_count[m, z, :]+self.gamma_dl[m,:])
##
##            lik += self.log_multi_beta(self.doc_sentiment_count[m,:]+self.alpha_lz)
##
##        lik -= n_docs * self.S * self.log_multi_beta(self.gamma_dl)
##        lik -= n_docs * self.log_multi_beta(self.alpha_lz)
##
##        return lik
##
##    def perplexity(self):
##        score = np.exp(-self.loglikelihood()/self.wordOccurenceMatrix.sum())
##        return score

    def get_top_sentiment_topic(self, topnums):
        '''每一个sentiment下的top-topic
           可以利用prettyTable进行显示
        '''
        table=PrettyTable()

        with open('C:/Users/Administrator/Desktop/data/评论/top_sentiment_topic_word.txt', 'w',encoding='utf-8') as f:
            for i in range(0, self.T):
                for j in range(self.S):
                    top_words = np.argsort(self.topic_sel_word[i,j :]).tolist()[:topnums]
                #print('输出的是每个topic的top_word的下标',top_words)
                    print(top_words)
                    top_word = [self.id2word[kk] for kk in top_words[0]]
                    table.add_column('sentiment{0}and topic_{1}'.format(i,j),top_word)
                    res = 'sentiment:{0},topic{1}:  \t {2} '.format(i,j,top_word)
                    f.write(res + '\n')
            f.close()
        print(table)
                # print(res)
    def get_sentiment_word(self,topnums):
        #得到每个sentiment下的word
        table=PrettyTable()
        with open('C:/Users/Administrator/Desktop/data/评论/top_sentiment_word','w',encoding='utf-8') as f:
            for i in range(self.S):
                top_words=np.argsort(self.sentiment_word_count[i,:]).tolist()[:topnums]
                #print(top_words)
                top_word=[self.id2word[j] for j in top_words]
                table.add_column('sentiment_{0}'.format(i),top_word)
                res='sentiment:{0} has topic_word  \t {1}'.format(i,top_word)
                f.write(res)
        print(table)

    def get_wordsentiment(self,sentiment_index,topnum=20):
        '''
        得到给定一个情感下的每个主题的词汇，以及概率
        只输出其中三个topic/的topnum的词汇
        sentiment_word
        利用
        :return:
        '''
        print('显示sentiment_index={0}下的3个主题的关键词汇'.format(sentiment_index))
        table=PrettyTable()
        word_list=[]
        for i in range(3):
            values_sentiment_topic=self.sentiment_topic_word[sentiment_index,i,:]
            max_list=np.argsort(values_sentiment_topic,).tolist()[-topnum:]
            ui=[]
            for id in max_list:
                ui.append((self.id2word[id],values_sentiment_topic[id]))
            #print(ui)
            word_list.append(ui)
            table.add_column('topic_{0}'.format(i),top_word)
        return  word_list

    def get_word_sentiment0(self,seta):
        '''
        得到每个词语的情感极性
        :return:self.word_sentiment_vocabulary
        '''
        # 主要使用sentiment-topic-word
        print('开始求解sentiment_word_list')
        sentiment_word_list={}
        word_sentiment_vocabulary={}
        for i in range(self.V):
            values=self.sentiment_topic_word[:,:,i]

            # 获取到了每一个主题下每一个词汇的值，根据其最大值所在位置，找到该所所在的情感
            m_index=np.argmax(values,axis=1)
            max_list=np.array([values[ii ,m_index[ii]] for ii in range(4)])
            max_index=np.argmax(max_list)
            word_sentiment_vocabulary[self.id2word[i]]=(max_index,max_list[max_index])
            if max_list[max_index] >=seta :
                if max_index not in sentiment_word_list.keys():
                    sentiment_word_list[max_index]=[]
                sentiment_word_list[max_index].append([self.id2word[i],max_list[max_index]])
            else:
                continue
        self.sentiment_word_list=sentiment_word_list
        self.word_sentiment_vocabulary=word_sentiment_vocabulary
        print('得到所有词汇利用主题模型得到的情感极性')
        print(len(self.sentiment_word_list[0])+len(self.sentiment_word_list[1])+
              len(self.sentiment_word_list[2]))
        return

    def get_topic_word(self, topnums):
        '''得到每个topic下的top-word'''

        table=PrettyTable()
        with open('C:/Users/Administrator/Desktop/data/评论/top_topic_word_sel','w',encoding='utf-8') as f:
            for i in range(self.T):
                top_words=np.argsort(self.topic_word_count[i,:]).tolist()[:topnums]
                print(top_words)
                top_word=[self.id2word[j] for j in top_words]
                table.add_column('topic_{0}'.format(i),top_word)
                res='topic:{0} has topic_word  \t {1}'.format(i,top_word)
                f.write(res)
        print(table)

    def pingu(self,emplsion,type):
        '''
        :param 使用topic coherence  对模型进行评价  umass metric  ----score d(j)-表示语料库中包含词汇j的个数  d(i,j)表示词汇i和j共同出现的词汇、
            其中词汇i以及j取自同一个情感标签
        :return:topic coherence
        '''
        coherence_umass=[]
        for i in range(self.S):
            word_list=[word_par[0] for word_par in self.sentiment_word_list[i]]
            coherence_umass.append(sum([self._scoreumass(i,j,emplsion) for i in word_list for j in word_list] ))
        self.coherence=sum(coherence_umass)
        print('umass_metric 下的topic coherence：{0}'.format(self.coherence ))
        return self.coherence

    def _scoreumass(self,i,j,emplsion):
        '''
        :param i: 词汇i
        :param j: 词汇j
        :param emplsion:
        :return: score
        '''
        count_j=0
        count_ij=0
        for doc in self.corpus:
            if j in doc:
                count_j+=1
            if j in doc and i in doc:
                count_ij+=1
        score =np.log2((count_ij+emplsion)/count_j)
        return score

    def print_doc_topic_word(self, doc_id, topic_list, word_nums=20):
        all_num = len(topic_list)
        table = PrettyTable()
        for i in topic_list:
            topword = np.argsort(self.topic_word[i, :])[:word_nums]
            table.add_column(i, [self.id2word[jj] for jj in topword])
        print(table)

        # 打印出来该文档上的主题分布以及在每个主题上面的个数的图形
        doc_topic_count = self.doc_topic_count[doc_id, :]
        sns.stripplot(x=list(range(0, all_num - 1)), y=doc_topic_count)
        for i in topic_list:
            sns.scatterplot(x=range(0, self.V - 1), y=self.topic_word[i, :])
            plt.show()
            sns.countplot(x=range(0, self.V - 1), hue=self.topic_word[i, :])
            plt.show()

def get_result(k,comment_train,sentiment_dict):
        M = ldamodel_sent(k, 4,0.01, 0.01, 0.01, comment_train, len(comment_train), 200, sentiment_dict)
        M.createdictionary()
        M.initial()
        start=time.time()
        M.gibbssampling()
        M.get_word_sentiment0(seta=0.01)
        M.pingu(emplsion=0.000001,type='umass_metric')
        end = time.time()
        print('gibbssampling stage use {0} second'.format(end - start))
        print(k,'完成')
        return M.coherence

if __name__ == '__main__':
    path1='C:/Users/Administrator/Desktop/data/corpus/handle_corpus_train.txt'
    print('开始加载数据')
    da=open(path1,encoding='utf-8').read()
    da1=da.lstrip('\ufeff')
    data=json.loads(da1)
    corpus_total=data['正面']+data['反面']

    sentiment_path='C:/Users/Administrator/Desktop/data/corpus/sentiment_corpus_train.txt'
    se = open(sentiment_path, encoding='utf-8').read()
    se = se.lstrip('\ufeff')
    sentiment_dict= json.loads(se)
    print('水杯评论加载完毕')

    corpus_cup=[]
    for doc in corpus_total:
        if len(doc) !=0:
            corpus_cup.append(doc)
    #comment_train, comment_test = train_test_split(corpus, test_size=0)
    comment_train=corpus_cup
    # cut_corpus是cut_comment
    #topic_list=[10,50,70,100]
    result=[]
    P = ldamodel_sent(5, 4, 0.01, 0.01, 0.01, comment_train, len(comment_train), 40, sentiment_dict)
    P.createdictionary()
    print('开始初始化')
    P.initial()
    print('初始化阶段的doc_sentiment_count')
    print(P.doc_sentiment_count)
    print(P.doc_count)
    start=time.time()
    P.gibbssampling()

##    from scipy.cluster.hierarchy import linkage, dendrogram, fcluster
##    import scipy.cluster.hierarchy as sch
##    from  sklearn.decomposition import PCA
##    pca=PCA()
##    x=M.topic_w
##    x_new=pca.fit_transform(x)
##    disMat = sch.distance.pdist(x_new,'euclidean')
##    Z = linkage(disMat,  method='ward')
##    P=sch.dendrogram(Z)
##    plt.title('Dendrogram for the Agglomerative Clustering')
##    plt.xlabel('sample_index')
##    plt.ylabel('distance')
##    plt.show()
##
##
##    labels_1 = fcluster(Z, t=1, criterion='inconsistent')
##    mark = ['or', 'ob', 'og', 'ok', '^r', '+r', 'sr', 'dr', '<r', 'pr','op','*p','*r','*b','*d','*k']
##    j = 0
##    for i in label_pred:
##        plt.plot([x_new[j,0]], [x_new[j,1]], mark[i], markersize = 5)
##        j +=1
##    topic_nums_new=np.max(labels_1)
##
##    #根据聚类结果将topic进行合并
##    m,n=x.shape
##    new_twc=x
##    twc_new=np.zeros([topic_nums_new,n])
##    for i in range(topic_nums_new):
##        ui=[j for j in range(len(labels_1)) if labels_1[j]==i+1]
##        print(ui)
##        for ji in ui:
##            twc_new[i,:]+=new_twc[ji,:]
##
##    #得到聚类之后的twc矩阵，进行聚类图展示
##    plt.scatter(twc_new[:,1],twc_new[:,0])
##
##    #相应的将stwc中的信息进行合并，最后显示各个情感下不同topic的Word信息 表
##    m,n,p=M.sentiment_topic_word_count.shape
##    m1,n1=M.sentiment_topic_count.shape
##    stc_new = coo_matrix((m,topic_nums_new)).toarray()
##    stwc_new=np.zeros([m,topic_nums_new,p])
##
##    for i in range(m):
##        for j in range(topic_nums_new):
##            ui=[jj for jj in range(len(labels_1)) if labels_1[jj]==j+1]
##            for ji in ui:
##                stwc_new[i,j,:]+=new_stwc[i,ji,:]
##            stc_new[i,j]=sum(stwc_new[i,j,:])
##
##
##
##    #得到sentiment_topic_word
##    stw_new=np.zeros((m,topic_nums_new,p))
##    for i in range(m):
##        for j in range(topic_nums_new):
##            for k in range(p):
##                stw_new[i, j, k] = (stwc_new[i, j, k] + 0.01)/(stc_new[i, j] + 0.01* V)

    #画表-得到每个sentiment下的topic 下的top10word以及概率

    table=PrettyTable()
    topnum=20
    id2word=P.id2word
    stw_new=P.sentiment_topic_word
    word_list=[]
    for i in range(3):
        values_sentiment_topic=stw_new[2,i,:]
        max_list=np.argsort(values_sentiment_topic,).tolist()[-topnum:]
        ui=[]
        for id in max_list:
            ui.append((id2word[id],values_sentiment_topic[id]))
        #print(ui)
        word_list.append(ui)
        table.add_column('topic_{0}'.format(i),ui)
    print(table)

    #得到其中几则评论的情感分布图

    test={} #test的key值为文档编号
    for i,doc in enumerate(comment_train):
        if len(doc)>=20:
            test[i]=doc
    #获得上面test中10则评论所对应的文档{-}情感分布

    ds=M.doc_sel
    test_dc=np.zeros((10,4))
    i=0
    for doc_id,doc in test.items():
        test_dc[i,:]=ds[doc_id,:]
        i+=1
        if i>=10:
            break

    #最后得到word-sentiment-vocabulary
    word_sentiment_vocabulary={}
    for i in range(p):
        values=stw_new[:,:,i]
        # 获取到了每一个主题下每一个词汇的值，根据其最大值所在位置，找到该所所在的情感
        m_index=np.argmax(values,axis=1)
        max_list=np.array([values[ii ,m_index[ii]] for ii in range(4)])
        max_index=np.argmax(max_list)
        word_sentiment_vocabulary[id2word[i]]=(max_index,max_list[max_index])

    #情感得分的计算

    sentence=set()
    for doc in corpus_cup:
        sentence.add(' '.join(doc))
    sentence=list(sentence)
    id2word=corpora.Dictionary(corpus_cup)
    word2id=id2word.token2id
    corpus = [id2word.doc2bow(text) for text in corpus0]
    model=models.TfidfModel(corpus,id2word=id2word)
    weight_corpus=model[corpus]

    def get_sentiment_score(pinglun,pinglun_index):
        jx_list=[]
        wordid_list=[]
        power=[]
        for word in pinglun :
            id_word=word2id[word]
            jx,par_jx=word_sentiment_vocabulary[word]
            if jx==0:
                power.append(1)
            elif jx==3:
                power.append(3)
            else:
                power.append(4)
            jx_list.append(jx)
            wordid_list.append(id_word)
        score=sum([power[i]*jx_list[i]*weight_corpus[pinglun_index][wordid_list[i]]  for i in range(len(pinglun))])
        return score

    #按照店铺整理数据
    path_info='C:/Users/Administrator/Desktop/data/评论/product_info_cup_before.csv'
    info_dianpu=pd.read_csv(path_info,engine='python',encoding='utf-8')

    ii=info_dianpu['shop_id'].value_counts()

    dianpu_info={}
    count_ii=ii.head(n=10)
    for i in range(len(count_ii)):
        product_id=info_dianpu[info_dianpu['shop_id']==count_ii.index[i]]['product_id'].values.tolist()
        shop_name=info_dianpu[info_dianpu['shop_id']==count_ii.index[i]]['shop_name'].unique()[0]
        if shop_name not in dianpu_info.keys():
            dianpu_info[shop_name]=[]
        dianpu_info[shop_name].extend(product_id)

    path='C:/Users/Administrator/Desktop/data/评论/comment_info_cup_final.csv'
    df_data=pd.read_csv(path,engine='python')
    oo=df_data['referenceId'].value_counts()
    pp=oo[oo>=300].index.tolist()
    corpus_dianpu={}
    for dianpu,product_id_list in dianpu_info.items():
        for product_id in product_id_list:
            if product_id in pp:
                ui = df_data[df_data['referenceId'] == product_id]['comment'].values.tolist()
            else:
                continue
            if dianpu not in corpus_dianpu.keys():
                corpus_dianpu[dianpu]=[]
            corpus_dianpu[dianpu].append(ui)

    from scipy.cluster.hierarchy import linkage, dendrogram, fcluster
    import scipy.cluster.hierarchy as sch
    from  sklearn.decomposition import PCA
    pca=PCA()
    x=new_twc
    x_new=pca.fit_transform(x)
    disMat = sch.distance.pdist(x_new,'euclidean')
    Z = linkage(disMat,  method='ward')
    P=sch.dendrogram(Z)
    plt.title('Dendrogram for the Agglomerative Clustering')
    plt.xlabel('sample_index')
    plt.ylabel('distance')
    plt.show()

    labels_1 = fcluster(Z, t=1, criterion='inconsistent')
##    mark = ['or', 'ob', 'og', 'ok', '^r', '+r', 'sr', 'dr', '<r', 'pr','op','*p','*r','*b','*d','*k']
##    j = 0
##    for i in label_pred:
##        plt.plot([x_new[j,0]], [x_new[j,1]], mark[i], markersize = 5)
##        j +=1
    topic_nums_new=np.max(labels_1)

    #根据聚类结果将topic进行合并
    m,n=x.shape
    new_twc=x
    twc_new=np.zeros([topic_nums_new,n])
    for i in range(topic_nums_new):
        ui=[j for j in range(len(labels_1)) if labels_1[j]==i+1]
        print(ui)
        for ji in ui:
            twc_new[i,:]+=new_twc[ji,:]

    #得到聚类之后的twc矩阵，进行聚类图展示
    plt.scatter(twc_new[:,1],twc_new[:,0])

    #相应的将stwc中的信息进行合并，最后显示各个情感下不同topic的Word信息 表
    m,n,p=M.sentiment_topic_word_count.shape
    m1,n1=M.sentiment_topic_count.shape
    stc_new = coo_matrix((m,topic_nums_new)).toarray()
    stwc_new=np.zeros([m,topic_nums_new,p])

    for i in range(m):
        for j in range(topic_nums_new):
            ui=[jj for jj in range(len(labels_1)) if labels_1[jj]==j+1]
            for ji in ui:
                stwc_new[i,j,:]+=new_stwc[i,ji,:]
            stc_new[i,j]=sum(stwc_new[i,j,:])

    #得到sentiment_topic_word
    stw_new=np.zeros((m,topic_nums_new,p))
    for i in range(m):
        for j in range(topic_nums_new):
            for k in range(p):
                stw_new[i, j, k] = (stwc_new[i, j, k] + 0.01)/(stc_new[i, j] + 0.01* V)

    #画表-得到每个sentiment下的topic 下的top10word以及概率

    table=PrettyTable()
    topnum=20
    id2word=M.id2word

    word_list=[]
    for i in range(3):
        values_sentiment_topic=stw_new[2,i,:]
        max_list=np.argsort(values_sentiment_topic,).tolist()[-topnum:]
        ui=[]
        for id in max_list:
            ui.append((id2word[id],values_sentiment_topic[id]))
        #print(ui)
        word_list.append(ui)
        table.add_column('topic_{0}'.format(i),ui)
    print(table)
    #
    test={}
    for i,doc in enumerate(comment_train):
        if len(doc)>=20:
            test[i]=doc
    doc_sel=np.zeros((M.D,4))
    for i in range(M.D):
            for j in range(M.S):
                doc_sel[i, j] = (M.doc_sentiment_count[i, j] + 0.01)/(M.doc_count[i] +M.S * 0.01)
    ds=M.doc_sel
    test_dc=np.zeros((10,4))
    i=0
    for doc_id,doc in test.items():
        test_dc[i,:]=ds[doc_id,:]
        i+=1
        if i>=10:
            break

    #最后得到word-sentiment-vocabulary
    word_sentiment_vocabulary={}
    for i in range(p):
        values=stw_new[:,:,i]
        # 获取到了每一个主题下每一个词汇的值，根据其最大值所在位置，找到该所所在的情感
        m_index=np.argmax(values,axis=1)
        max_list=np.array([values[ii ,m_index[ii]] for ii in range(4)])
        max_index=np.argmax(max_list)
        word_sentiment_vocabulary[id2word[i]]=(max_index,max_list[max_index])

    #情感得分的计算

    sentence=set()
    for doc in corpus_cup:
        sentence.add(' '.join(doc))
    sentence=list(sentence)
    id2word=corpora.Dictionary(corpus_cup)
    word2id=id2word.token2id
    corpus = [id2word.doc2bow(text) for text in corpus0]
    model=models.TfidfModel(corpus,id2word=id2word)
    weight_corpus=model[corpus]

    def get_sentiment_score(pinglun,pinglun_index):
        jx_list=[]
        wordid_list=[]
        power=[]
        for word in pinglun :
            id_word=word2id[word]
            jx,par_jx=word_sentiment_vocabulary[word]
            if jx==0:
                power.append(1)
            elif jx==3:
                power.append(3)
            else:
                power.append(4)
            jx_list.append(jx)
            wordid_list.append(id_word)
        score=sum([power[i]*jx_list[i]*weight_corpus[pinglun_index][wordid_list[i]]  for i in range(len(pinglun))])
        return score

    #按照店铺整理数据
    path_info='C:/Users/Administrator/Desktop/data/评论/product_info_cup_before.csv'
    info_dianpu=pd.read_csv(path_info,engine='python',encoding='utf-8')

    ii=info_dianpu['shop_id'].value_counts()

    dianpu_info={}
    count_ii=ii.head(n=10)
    for i in range(len(count_ii)):
        product_id=info_dianpu[info_dianpu['shop_id']==count_ii.index[i]]['product_id'].values.tolist()
        shop_name=info_dianpu[info_dianpu['shop_id']==count_ii.index[i]]['shop_name'].unique()[0]
        if shop_name not in dianpu_info.keys():
            dianpu_info[shop_name]=[]
        dianpu_info[shop_name].extend(product_id)

    path='C:/Users/Administrator/Desktop/data/评论/comment_info_cup_final.csv'
    df_data=pd.read_csv(path,engine='python')
    oo=df_data['referenceId'].value_counts()
    pp=oo[oo>=300].index.tolist()
    corpus_dianpu={}
    for dianpu,product_id_list in dianpu_info.items():
        for product_id in product_id_list:
            if product_id in pp:
                ui = df_data[df_data['referenceId'] == product_id]['comment'].values.tolist()
            else:
                continue
            if dianpu not in corpus_dianpu.keys():
                corpus_dianpu[dianpu]=[]
            corpus_dianpu[dianpu].append(ui)


import json

import time
import argparse
from tqdm import tqdm

def prepare_train_data(file_input='./data/crowd/train.json', file_output='./data/processed/train.txt'):
    '''
    @param file_input (str): input filename
    @param file_output (str): output text file, model inputs
    '''
    f_in = open(file_input, 'r', encoding='utf-8')
    f_out = open(file_output, 'w', encoding='utf-8')
    sentence_left = []
    mention = []
    sentence_right = []
    text_output = []
    for i, line in tqdm(enumerate(f_in)):
        x = json.loads(line)
        sentence = ''
        y_list = x.get('y_str', None)
        sentence_left = x.get('left_context_token', None)
        mention = x.get('mention_span', None)
        sentence_right = x.get('right_context_token', None)
        if y_list is not None:
            for word in y_list:
                sentence += word + ' '
            sentence = sentence[:-1]
            sentence = sentence + '\t'
            for word in sentence_left:
                sentence += word + ' '
            sentence += mention
            for word in sentence_right:
                sentence += word + ' '
            sentence += '[SEP] '
            sentence += mention + '\n'
            text_output.append(sentence)

    for line in text_output:
        f_out.writelines(line)
    f_in.close()
    f_out.close()

# create 6 different patterns of masked data
def prepare_masked_data(file_input='./data/crowd/train.json', file_output='./data/processed/train.txt'):
    '''
    @param file_input (str): input filename
    @param file_output (str): output text file, model inputs
    '''

    f_in = open(file_input, 'r', encoding='utf-8')
    sentence_left = []
    mention = []
    sentence_right = []
    text_output = []
    patterns = [
        'and any other [MASK] ',
        # 'and some other [MASK] ',
        # '[MASK] such as ',
        # 'such [MASK] as ',
        # '[MASK] including ',
        # '[MASK] especially '
    ]
    f_out = []
    iter = len(patterns)
    for i in range(iter):
        f_out.append(open(file_output + '.' + 'pattern_' + str(i), 'w', encoding='utf-8'))

    for i, line in tqdm(enumerate(f_in)):
        x = json.loads(line)
        y_list = x.get('y_str', None)
        sentence_left = x.get('left_context_token', None)
        mention = x.get('mention_span', None)
        sentence_right = x.get('right_context_token', None)
        # write
        for p in range(iter):
            sentence = ''
            if y_list is not None:
                for word in y_list:
                    sentence += word + ' '
                sentence = sentence[:-1]
                sentence = sentence + '\t'
                for word in sentence_left:
                    sentence += word + ' '
                if p == 0 or p == 1:
                    sentence += mention + ' '
                    sentence += patterns[p]
                else:
                    sentence += patterns[p]
                    sentence += mention + ' '
                for word in sentence_right:
                    sentence += word + ' '
                sentence = sentence[:-1]
                sentence += '\n'
                f_out[p].writelines(sentence)

    for i in range(iter):
        f_out[i].close()
    f_in.close()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Arguments for preparing data.')
    parser.add_argument('-i', type=str, default='./data/distant.json', help='input file')
    parser.add_argument('-o', type=str, default='./data/train_distant.txt', help='output file')
    parser.add_argument('-m', type=int, default=0,
                        help='generation mode: 0 - for training data; 1 - for masked training data')
    args = parser.parse_args()
    print('Reformatting data...')
    start = time.time()
    if args.m != '':
        if args.m == 0:
            prepare_train_data(args.i, args.o)
        elif args.m == 1:
            prepare_masked_data(args.i, args.o)
        else:
            print('Arg error.')
    else:
        print('Arg error.')

    # debug
    # prepare_data('./data/crowd/dev.json', './data/processed/dev.txt')
    # prepare_masked_data('./data/crowd/dev.json', './data/processed/dev.txt')

    end = time.time()
    print('Loading finished:\t', end - start)


# -*- coding: utf-8 -*-
from __future__ import absolute_import

import cPickle
import os
import shutil

from django.conf import settings
from django.core.files.base import ContentFile
from django.core.mail import EmailMultiAlternatives
from django.db import DatabaseError
from django.template import Context
from django.template.loader import get_template
from django.utils import timezone

from celery import Task
from celery.utils.log import get_task_logger

from jobs.presets import TagParser
from utils import (
    garmin, kml, osmand, osmconf, osmparse, overpass, pbf, shp, thematic_shp
)

# Get an instance of a logger
logger = get_task_logger(__name__)

# ExportTask abstract base class and subclasses.

class ExportTask(Task):
    """
    Abstract base class for export tasks.
    """

    # whether to abort the whole run if this task fails.
    abort_on_error = False

    class Meta:
        abstract = True

    def on_success(self, retval, task_id, args, kwargs):
        """
        Update the successfuly completed task as follows:

            1. update the time the task completed
            2. caclulate the size of the output file
            3. calculate the download path of the export
            4. create the export download directory
            5. copy the export file to the download directory
            6. create the export task result
            7. update the export task status and save it
        """
        from tasks.models import ExportTask, ExportTaskResult
        # update the task
        finished = timezone.now()
        task = ExportTask.objects.get(celery_uid=task_id)
        task.finished_at = finished
        # get the output
        output_url = retval['result']
        stat = os.stat(output_url)
        size = stat.st_size / 1024 / 1024.00
        # construct the download_path
        download_root = settings.EXPORT_DOWNLOAD_ROOT
        parts = output_url.split('/')
        filename = parts[-1]
        run_uid = parts[-2]
        run_dir = '{0}{1}'.format(download_root, run_uid)
        download_path = '{0}{1}/{2}'.format(download_root, run_uid, filename)
        try:
            if not os.path.exists(run_dir):
                os.makedirs(run_dir)
            # don't copy raw overpass data
            if (task.name != 'OverpassQuery'):
                shutil.copy(output_url, download_path)
        except IOError as e:
            logger.error('Error copying output file to: {0}'.format(download_path))
        # construct the download url
        download_media_root = settings.EXPORT_MEDIA_ROOT
        download_url = '{0}{1}/{2}'.format(download_media_root, run_uid, filename)
        # save the task and task result
        result = ExportTaskResult(
            task=task,
            filename=filename,
            size=size,
            download_url=download_url
        )
        result.save()
        task.status = 'SUCCESS'
        task.save()

    def on_failure(self, exc, task_id, args, kwargs, einfo):
        """
        Update the failed task as follows:

            1. pull out the ExportTask
            2. update the task status and finish time
            3. create an export task exception
            4. save the export task with the task exception
            5. run ExportTaskErrorHandler if the run should be aborted
               - this is only for initial tasks on which subsequent export tasks depend
        """
        from tasks.models import ExportTask, ExportTaskException, ExportRun
        logger.debug('Task name: {0} failed, {1}'.format(self.name, einfo))
        task = ExportTask.objects.get(celery_uid=task_id)
        task.status = 'FAILED'
        task.finished_at = timezone.now()
        task.save()
        exception = cPickle.dumps(einfo)
        ete = ExportTaskException(task=task, exception=exception)
        ete.save()
        if self.abort_on_error:
            run = ExportRun.objects.get(tasks__celery_uid=task_id)
            run.status = 'FAILED'
            run.finished_at = timezone.now()
            run.save()
            error_handler = ExportTaskErrorHandler()
            # run error handler
            stage_dir = kwargs['stage_dir']
            error_handler.si(run_uid=str(run.uid), task_id=task_id, stage_dir=stage_dir).delay()

    def after_return(self, *args, **kwargs):
        logger.debug('Task returned: {0}'.format(self.request))

    def update_task_state(self, run_uid=None, name=None):
        """
        Update the task state and celery task uid.
        Can use the celery uid for diagnostics.
        """
        started = timezone.now()
        from tasks.models import ExportTask
        celery_uid = self.request.id
        try:
            task = ExportTask.objects.get(run__uid=run_uid, name=name)
            celery_uid = self.request.id
            task.celery_uid = celery_uid
            task.status = 'RUNNING'
            task.started_at = started
            task.save()
            logger.debug('Updated task: {0} with uid: {1}'.format(task.name, task.uid))
        except DatabaseError as e:
            logger.error('Updating task {0} state throws: {1}'.format(task.name, e))
            raise e

class OSMConfTask(ExportTask):
    """
    Task to create the ogr2ogr conf file.
    """
    name = 'OSMConf'
    abort_on_error = True

    def run(self, run_uid=None, categories=None, stage_dir=None, job_name=None):
        self.update_task_state(run_uid=run_uid, name=self.name)
        conf = osmconf.OSMConfig(categories, job_name=job_name)
        configfile = conf.create_osm_conf(stage_dir=stage_dir)
        return {'result': configfile}

class OverpassQueryTask(ExportTask):
    """
    Class to run an overpass query.
    """
    name = 'OverpassQuery'
    abort_on_error = True

    def run(self, run_uid=None, stage_dir=None, job_name=None, filters=None, bbox=None):
        """
        Runs the query and returns the path to the filtered osm file.
        """
        self.update_task_state(run_uid=run_uid, name=self.name)
        op = overpass.Overpass(
            bbox=bbox, stage_dir=stage_dir,
            job_name=job_name, filters=filters
        )
        op.run_query()  # run the query
        filtered_osm = op.filter()  # filter the results
        return {'result': filtered_osm}

class OSMToPBFConvertTask(ExportTask):
    """
    Task to convert osm to pbf format.
    Returns the path to the pbf file.
    """
    name = 'OSM2PBF'
    abort_on_error = True

    def run(self, run_uid=None, stage_dir=None, job_name=None):
        self.update_task_state(run_uid=run_uid, name=self.name)
        osm = stage_dir + job_name + '.osm'
        pbffile = stage_dir + job_name + '.pbf'
        o2p = pbf.OSMToPBF(osm=osm, pbffile=pbffile)
        pbffile = o2p.convert()
        return {'result': pbffile}

class OSMPrepSchemaTask(ExportTask):
    """
    Task to create the default sqlite schema.
    """
    name = 'OSMSchema'
    abort_on_error = True

    def run(self, run_uid=None, stage_dir=None, job_name=None):
        self.update_task_state(run_uid=run_uid, name=self.name)
        osm = stage_dir + job_name + '.pbf'
        sqlite = stage_dir + job_name + '.sqlite'
        osmconf = stage_dir + job_name + '.ini'
        osmparser = osmparse.OSMParser(osm=osm, sqlite=sqlite, osmconf=osmconf)
        osmparser.create_spatialite()
        osmparser.create_default_schema()
        osmparser.update_zindexes()
        return {'result': sqlite}

class ThematicLayersExportTask(ExportTask):
    """
    Task to export thematic shapefile.
    """

    name = "Thematic Shapefile Export"

    def run(self, run_uid=None, stage_dir=None, job_name=None):
        from tasks.models import ExportRun
        self.update_task_state(run_uid=run_uid, name=self.name)
        run = ExportRun.objects.get(uid=run_uid)
        tags = run.job.categorised_tags
        sqlite = stage_dir + job_name + '.sqlite'
        try:
            t2s = thematic_shp.ThematicSQliteToShp(sqlite=sqlite, tags=tags, job_name=job_name)
            t2s.generate_thematic_schema()
            out = t2s.convert()
            return {'result': out}
        except Exception as e:
            logger.error('Raised exception in thematic task, %s', str(e))
            raise Exception(e)  # hand off to celery..

class ShpExportTask(ExportTask):
    """
    Class defining SHP export function.
    """
    name = 'Default Shapefile Export'

    def run(self, run_uid=None, stage_dir=None, job_name=None):
        self.update_task_state(run_uid=run_uid, name=self.name)
        sqlite = stage_dir + job_name + '.sqlite'
        shapefile = stage_dir + job_name + '_shp'
        try:
            s2s = shp.SQliteToShp(sqlite=sqlite, shapefile=shapefile)
            out = s2s.convert()
            return {'result': out}
        except Exception as e:
            logger.error('Raised exception in shapefile export, %s', str(e))
            raise Exception(e)

class KmlExportTask(ExportTask):
    """
    Class defining KML export function.
    """
    name = 'KML Export'

    def run(self, run_uid=None, stage_dir=None, job_name=None):
        self.update_task_state(run_uid=run_uid, name=self.name)
        sqlite = stage_dir + job_name + '.sqlite'
        kmlfile = stage_dir + job_name + '.kml'
        try:
            s2k = kml.SQliteToKml(sqlite=sqlite, kmlfile=kmlfile)
            out = s2k.convert()
            return {'result': out}
        except Exception as e:
            logger.error('Raised exception in kml export, %s', str(e))
            raise Exception(e)

class ObfExportTask(ExportTask):
    """
    Class defining OBF export function.
    """
    name = 'OBF Export'

    def run(self, run_uid=None, stage_dir=None, job_name=None):
        self.update_task_state(run_uid=run_uid, name=self.name)
        pbffile = stage_dir + job_name + '.pbf'
        map_creator_dir = settings.OSMAND_MAP_CREATOR_DIR
        work_dir = stage_dir + 'osmand'
        try:
            o2o = osmand.OSMToOBF(
                pbffile=pbffile, work_dir=work_dir, map_creator_dir=map_creator_dir
            )
            out = o2o.convert()
            obffile = stage_dir + job_name + '.obf'
            shutil.move(out, obffile)
            shutil.rmtree(work_dir)
            return {'result': obffile}
        except Exception as e:
            logger.error('Raised exception in obf export, %s', str(e))
            raise Exception(e)

class SqliteExportTask(ExportTask):
    """
    Class defining SQLITE export function.
    """

    name = 'SQLITE Export'

    def run(self, run_uid=None, stage_dir=None, job_name=None):
        self.update_task_state(run_uid=run_uid, name=self.name)
        # sqlite already generated by OSMPrepSchema so just return path.
        sqlite = stage_dir + job_name + '.sqlite'
        return {'result': sqlite}

class GarminExportTask(ExportTask):
    """
    Class defining GARMIN export function.
    """

    name = 'Garmin Export'
    _region = ''  # set by the task_runner

    @property
    def region(self,):
        return self._region

    @region.setter
    def region(self, value):
        self._region = value

    def run(self, run_uid=None, stage_dir=None, job_name=None):
        self.update_task_state(run_uid=run_uid, name=self.name)
        work_dir = stage_dir + 'garmin'
        config = settings.GARMIN_CONFIG  # get path to garmin config
        pbffile = stage_dir + job_name + '.pbf'
        try:
            o2i = garmin.OSMToIMG(
                pbffile=pbffile, work_dir=work_dir,
                config=config, region=None, debug=False
            )
            o2i.run_splitter()
            out = o2i.run_mkgmap()
            imgfile = stage_dir + job_name + '_garmin.zip'
            shutil.move(out, imgfile)
            shutil.rmtree(work_dir)
            return {'result': imgfile}
        except Exception as e:
            logger.error('Raised exception in garmin export, %s', str(e))
            raise Exception(e)

class GeneratePresetTask(ExportTask):
    """
    Generates a JOSM Preset from the exports selected features.
    """

    name = 'Generate Preset'

    def run(self, run_uid=None, stage_dir=None, job_name=None):
        from tasks.models import ExportRun
        from jobs.models import ExportConfig
        self.update_task_state(run_uid=run_uid, name=self.name)
        run = ExportRun.objects.get(uid=run_uid)
        job = run.job
        user = job.user
        feature_save = job.feature_save
        feature_pub = job.feature_pub
        # check if we should create a josm preset
        if feature_save or feature_pub:
            tags = job.tags.all()
            tag_parser = TagParser(tags=tags)
            xml = tag_parser.parse_tags()
            preset_file = ContentFile(xml)
            name = job.name
            filename = job_name + '_preset.xml'
            content_type = 'application/xml'
            config = ExportConfig.objects.create(
                name=name, filename=filename,
                config_type='PRESET', content_type=content_type,
                user=user, published=feature_pub
            )
            config.upload.save(filename, preset_file)

            output_path = config.upload.path
            job.configs.clear()
            job.configs.add(config)
            return {'result': output_path}

class FinalizeRunTask(Task):
    """
    Finalizes export run.

    Cleans up staging directory.
    Updates run with finish time.
    Emails user notification.
    """

    name = 'Finalize Export Run'

    def run(self, run_uid=None, stage_dir=None):
        from tasks.models import ExportRun
        run = ExportRun.objects.get(uid=run_uid)
        run.status = 'COMPLETED'
        tasks = run.tasks.all()
        # mark run as incomplete if any tasks fail
        for task in tasks:
            if task.status == 'FAILED':
                run.status = 'INCOMPLETE'
        finished = timezone.now()
        run.finished_at = finished
        run.save()
        try:
            shutil.rmtree(stage_dir)
        except IOError as e:
            logger.error('Error removing {0} during export finalize'.format(stage_dir))

        # send notification email to user
        hostname = settings.HOSTNAME
        url = 'http://{0}/exports/{1}'.format(hostname, run.job.uid)
        addr = run.user.email
        subject = "Your HOT Export is ready"
        to = [addr]
        from_email = 'HOT Exports <exports@hotosm.org>'
        ctx = {
            'url': url,
            'status': run.status
        }
        text = get_template('email/email.txt').render(Context(ctx))
        html = get_template('email/email.html').render(Context(ctx))
        msg = EmailMultiAlternatives(subject, text, to=to, from_email=from_email)
        msg.attach_alternative(html, "text/html")
        msg.send()

class ExportTaskErrorHandler(Task):
    """
    Handles un-recoverable errors in export tasks.
    """

    name = "Export Task Error Handler"

    def run(self, run_uid, task_id=None, stage_dir=None):
        from tasks.models import ExportRun
        finished = timezone.now()
        run = ExportRun.objects.get(uid=run_uid)
        run.finished_at = finished
        run.status = 'FAILED'
        run.save()
        try:
            if os.path.isdir(stage_dir):
                #leave the stage_dir in place for debugging
                #shutil.rmtree(stage_dir)
                pass
        except IOError as e:
            logger.error('Error removing {0} during export finalize'.format(stage_dir))
        hostname = settings.HOSTNAME
        url = 'http://{0}/exports/{1}'.format(hostname, run.job.uid)
        addr = run.user.email
        subject = "Your HOT Export Failed"
        # email user and administrator
        to = [addr, settings.TASK_ERROR_EMAIL]
        from_email = 'HOT Exports <exports@hotosm.org>'
        ctx = {
            'url': url,
            'task_id': task_id
        }
        text = get_template('email/error_email.txt').render(Context(ctx))
        html = get_template('email/error_email.html').render(Context(ctx))
        msg = EmailMultiAlternatives(subject, text, to=to, from_email=from_email)
        msg.attach_alternative(html, "text/html")
        msg.send()


'''
    File:   stepping_exp.py
    Author: Nicholas Mattei (nsmattei@gmail.com)
    Date:   July 30th, 2015

  * Copyright (c) 2015, Nicholas Mattei and NICTA
  * All rights reserved.
  *
  * Developed by: Nicholas Mattei
  *               NICTA
  *               http://www.nickmattei.net
  *               http://www.preflib.org
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *     * Redistributions of source code must retain the above copyright
  *       notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above copyright
  *       notice, this list of conditions and the following disclaimer in the
  *       documentation and/or other materials provided with the distribution.
  *     * Neither the name of NICTA nor the
  *       names of its contributors may be used to endorse or promote products
  *       derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY NICTA ''AS IS'' AND ANY
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED. IN NO EVENT SHALL NICTA BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

About
--------------------
  This runs a simple stepping experiment and saves the results to a file.
  Note that it DOES NOT TRACK the score vector etc.  This is meant
  as a way to run a series of steps -- not a comprehensive experimental framework.

'''
import pickle
import numpy as np
import random
import itertools
import pandas as pd
from collections import Counter

_DEBUG = False

from peerselect import impartial
from peerselect import profile_generator

# Set the Seed...
# random.seed(15)

class Impartial:
    VANILLA = "Vanilla"
    PARTITION = "Partition"
    DOLLAR = "DollarPartition"
    DPR = "DollarPartitionRaffle"
    CREDIABLE = "CredibleSubset"
    RAFFLE = "DollarRaffle"
    ALL = (VANILLA, DOLLAR, PARTITION, RAFFLE, CREDIABLE, DPR)

# Exponential.
#scores = [pow(n, 4), pow(n, 3), pow(n, 2), n, 1]
#dist = [0.1, 0.2, 0.2, 0.2, 0.3]

#Borda
#scores = [3, 2, 1, 0]
#dist = [0.25, 0.25, 0.25, 0.25]

# Psuedo 10 point Normal... \sigma~=1
scores = [8, 7, 6, 5, 4, 3, 2, 1, 0]
# Normal...
dist = [0.03, 0.05, 0.12, 0.15, 0.30, 0.15, 0.12, 0.05, 0.03]

s = 1000
test_n = [130]
test_k = [25]
test_m = [10, 20, 40]
test_l = [5]
test_p = [0.1]

# Output File name
## Save the current runs
out_name = "../notebooks/pickled_runs/pub_NSF1000s_130n_25k_10-40m_5l.pickle"

# Map for all results... We'll build a high level index out of this later...
results = {}
for n,k,m,l,p in itertools.product(test_n, test_k, test_m, test_l, test_p):

    # Compute some artifacts from the scoring distributions.
    agents = np.arange(0, n)

    #Bit Hacky but faster.... Generate a unanimous score matrix and compute some properties..
    t_profile = profile_generator.generate_mallows_mixture_profile(agents, agents, [1.0], [agents], [0.0])
    t_matrix = profile_generator.profile_classes_to_score_matrix(t_profile, scores, dist)

    #Determine how many of each bin we have.
    # Compute the binning from The SCORE MATRIX ---> Tally guy 0.
    t = Counter(list(t_matrix[:,0]))
    size = [t[k] for k in sorted(t.keys(), reverse=True)]
    # Determine how many of each bin we should have...
    n_from_bin = [0]*len(size)
    left = k
    for i,j in enumerate(n_from_bin):
        if left > 0: n_from_bin[i] = min(size[i], left)
        left -= size[i]
    cum_n_from_bin = list(np.cumsum(n_from_bin))

    # Determine what bin they should go in according to the ground truth.
    # Just take the first guys's vector and iterate over it.
    # Guy i got score v and should be in the corresponding bin as indexed by the score vector.
    in_bin = {i:scores.index(v) for i,v in enumerate(list(t_matrix[:, 0]))}

    # Containers for Results
    count_results = {x:[0]*k for x in Impartial.ALL}
    bin_results = {x:[0]*len(size) for x in Impartial.ALL}
    for c_sample in range(s):
        #Generate a full profile and a clustering.
        profile = profile_generator.generate_mallows_mixture_profile(agents, agents, [1.0], [agents], [p])
        clustering = impartial.even_partition_order(sorted(agents, key=lambda j: random.random()), l)

        #Generate an approx-m-regular assignment.
        m_assignment = profile_generator.generate_approx_m_regular_assignment(agents, m, clustering, randomize=True)
        score_matrix = profile_generator.profile_classes_to_score_matrix(profile, scores, dist)
        score_matrix = profile_generator.restrict_score_matrix(score_matrix, m_assignment)

        #Compute Target Set.
        target_set = impartial.vanilla(score_matrix, k)

        ws = {}
        ws[Impartial.DOLLAR] = impartial.dollar_partition_explicit(score_matrix, k, clustering, normalize=True)
        size_ws = len(ws[Impartial.DOLLAR])
        # Let everyone else have the same size set so they are all compareable.
        ws[Impartial.VANILLA] = [i for i,j in impartial.vanilla(score_matrix, size_ws)]
        # Let CRED, PART, and RAFFLE have bigger sets...
        ws[Impartial.PARTITION] = impartial.partition_explicit(score_matrix, size_ws, clustering, normalize=False)
        ws[Impartial.CREDIABLE] = impartial.credible_subset(score_matrix, size_ws, m, normalize=False)
        ws[Impartial.DPR] = impartial.dollar_raffle_explicit(score_matrix, size_ws, clustering, normalize=True)
        #Call Raffle and have everyone in a cluster by themselves = Dollar.
        ws[Impartial.RAFFLE] = impartial.dollar_raffle(score_matrix, size_ws, n, randomize=True, normalize=True)

        # Update the Per position information.
        for i,tup in enumerate(target_set):
            a = tup[0]
            for x in Impartial.ALL:
                if a in ws[x]: count_results[x][i] += 1

        # Update the per bin picking for each type.
        for x in Impartial.ALL:
            for e in ws[x]:
                bin_results[x][in_bin[e]] += 1

    # Make cumulative versions for easy graphing...
    cum_count_results = {x:[0]*k for x in Impartial.ALL}
    cum_bin_results = {x:[0]*len(size) for x in Impartial.ALL}
    for x in Impartial.ALL:
        cum_count_results[x] = [v/float((i+1.) * s) for i,v in enumerate(list(np.cumsum(count_results[x])))]
        cum_bin_results[x] = [v/float(cum_n_from_bin[i] * s) for i,v in enumerate(np.cumsum(bin_results[x]))]
        # Normalize the counts and bins by n Samples to get a rate.
        count_results[x] = [float(i) / float(s) for i in count_results[x]]
        bin_results[x] = [float(i) / float(s) for i in bin_results[x]]

    # This should likely be some kind of multiindex but I can't figure it out.
    t = (n, k, m, l, p, "count")
    results[t] = pd.DataFrame(count_results)
    results[t].index += 1
    t = (n, k, m, l, p, "cum_count")
    results[t] = pd.DataFrame(cum_count_results)
    results[t].index += 1
    t = (n, k, m, l, p, "bin")
    results[t] = pd.DataFrame(bin_results)
    results[t].index += 1
    t = (n, k, m, l, p, "cum_bin")
    results[t] = pd.DataFrame(cum_bin_results)
    results[t].index += 1
    print("Finished: " + ",".join([str(x) for x in [n, k, m, l, p]]))

## Save the current runs
with open(out_name, 'wb') as output_file:
    pickle.dump(results, output_file)

print("Done")
print("Wrote to: " + out_name)
print("Score: " + str(scores))
print("Distribution: " + str(dist))
print("Size: " + str(size))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
2021.09.18
author: yasin sahin
written to construct lda algorithm together with logistic regression

"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix, accuracy_score

# importing dataset
dataset = pd.read_csv('Wine.csv')
x = dataset.iloc[:,:-1].values
y = dataset.iloc[:,-1].values

# splitting training and test set
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 0)

# Scaling independent variables
sc_x = StandardScaler()
x_train = sc_x.fit_transform(x_train)
x_test = sc_x.transform(x_test)

# applying PCA and reducing dimension to two
lda = LinearDiscriminantAnalysis(n_components = 2) # initiliazing pca instant
x_train = lda.fit_transform(x_train, y_train)
x_test = lda.transform(x_test)

# fitting logistic regression classification model to dataset
classifier = LogisticRegression(random_state = 0)
classifier.fit(x_train, y_train)

# predicting test set results and calculating confusion matrix and accuracy
y_pred = classifier.predict(x_test)
c_m = confusion_matrix(y_test, y_pred)
accuracy = accuracy_score(y_test, y_pred)
print(f'Accuracy is: {accuracy}')

# visualizing training results
x_set, y_set = x_train, y_train
x_1, x_2 = np.meshgrid(np.arange(start = x_set[:,0].min() - 1, stop = x_set[:,0].max() + 1, step = 0.01), \
                       np.arange(start = x_set[:,1].min() - 1, stop = x_set[:,1].max() + 1, step = 0.01))

plt.contourf(x_1, x_2, classifier.predict(np.array([x_1.ravel(),x_2.ravel()]).T).reshape(x_1.shape), alpha = 0.75, cmap = ListedColormap(('red','green','blue')))
plt.xlim(x_1.min(),x_1.max())
plt.ylim(x_2.min(),x_2.max())
for i,j in enumerate(np.unique(y_set)):
    plt.scatter(x_set[y_set==j,0],x_set[y_set==j,1], c = ListedColormap(('red','green','blue'))(i), label =j)
plt.title('Logistic Regression (training results)')
plt.xlabel('LD1')
plt.ylabel('LD2')
plt.legend()
plt.show()

# visualizing test results
x_set, y_set = x_test, y_test
x_1, x_2 = np.meshgrid(np.arange(start = x_set[:,0].min() - 1, stop = x_set[:,0].max() + 1, step = 0.01), \
                       np.arange(start = x_set[:,1].min() - 1, stop = x_set[:,1].max() + 1, step = 0.01))

plt.contourf(x_1, x_2, classifier.predict(np.array([x_1.ravel(),x_2.ravel()]).T).reshape(x_1.shape), alpha = 0.75, cmap = ListedColormap(('red','green','blue')))
plt.xlim(x_1.min(),x_1.max())
plt.ylim(x_2.min(),x_2.max())
for i,j in enumerate(np.unique(y_set)):
    plt.scatter(x_set[y_set==j,0],x_set[y_set==j,1], c = ListedColormap(('red','green','blue'))(i), label =j)
plt.title('Logistic Regression (test results)')
plt.xlabel('LD1')
plt.ylabel('LD2')
plt.legend()
plt.show()


"""Inspiration:
https://github.com/tommylees112/esowc_notes/blob/master/src/plotting_utils.py
"""
import warnings

import numpy as np
import pandas as pd
import xarray as xr
import shapely

from scipy import stats
from scipy.stats import pearsonr

import matplotlib.pyplot as plt

import cartopy
from cartopy.mpl.ticker import LongitudeFormatter, LatitudeFormatter
from mpl_toolkits.axes_grid1.inset_locator import inset_axes

def plot_xarray_on_globe(da, borders=True, coastlines=True, **kwargs):
    """ Plot the LOCATION of an xarray object """
    # get the center points for the maps
    mid_lat = np.mean(da.lat.values)
    mid_lon = np.mean(da.lon.values)
    # create the base layer
    fig = plt.figure(figsize=(12, 8))
    ax = fig.add_subplot(1, 1, 1, projection=cartopy.crs.Orthographic(mid_lon, mid_lat))
    # ax = plt.axes(projection=cartopy.crs.Orthographic(mid_lon, mid_lat))

    vmin = kwargs.pop("vmin", None)
    vmax = kwargs.pop("vmax", None)
    cmap = kwargs.pop("cmap", None)
    da.plot(ax=ax, transform=cartopy.crs.PlateCarree(), vmin=vmin, vmax=vmax, cmap=cmap)

    ax.coastlines()
    ax.add_feature(cartopy.feature.BORDERS, linestyle=":")
    ax.add_feature(cartopy.feature.LAKES, facecolor=None)
    fig = plt.gcf()
    ax.outline_patch.set_visible(False)
    return fig, ax

def plot_xarray_on_map(da, borders=True, coastlines=True, **kwargs):
    """ Plot the LOCATION of an xarray object """

    # create the base layer
    fig = plt.figure(figsize=(12, 8))
    ax = fig.add_subplot(1, 1, 1, projection=cartopy.crs.Robinson())
    # ax = plt.axes(projection=cartopy.crs.Orthographic(mid_lon, mid_lat))

    vmin = kwargs.pop("vmin", None)
    vmax = kwargs.pop("vmax", None)
    cmap = kwargs.pop("cmap", None)
    robust = kwargs.pop("robust", None)
    da.plot(
        ax=ax,
        transform=cartopy.crs.PlateCarree(),
        vmin=vmin,
        vmax=vmax,
        cmap=cmap,
        robust=robust,
        cbar_kwargs={"shrink": 0.4},
    )

    ax.coastlines()
    ax.add_feature(cartopy.feature.BORDERS, linestyle=":")
    ax.add_feature(cartopy.feature.LAKES, facecolor=None)
    fig = plt.gcf()
    ax.outline_patch.set_visible(False)
    return fig, ax

def add_point_location_to_map(point, ax, color=(0, 0, 0, 1), **kwargs):
    """ """
    assert isinstance(
        point, shapely.geometry.point.Point
    ), f"point should be of type shapely.geometry.point.Point. Currently: {type(point)}"
    # assert isinstance(ax, cartopy.mpl.geoaxes.GeoAxesSubplot), f"Axes need to be cartopy.mpl.geoaxes.GeoAxesSubplot. Currently: {type(ax)}"
    ax.scatter(
        point.x, point.y, transform=cartopy.crs.PlateCarree(), c=[color], **kwargs
    )

    return

def add_points_to_map(ax, geodf, point_colors="#0037ff"):
    """ Add the point data stored in `geodf.geometry` as points to ax
    Arguments:
    ---------
    : geodf (geopandas.GeoDataFrame)
        gpd.GeoDataFrame with a `geometry` column containing shapely.Point geoms
    : ax (cartopy.mpl.geoaxes.GeoAxesSubplot)
    """
    assert isinstance(
        ax, cartopy.mpl.geoaxes.GeoAxesSubplot
    ), f"Axes need to be cartopy.mpl.geoaxes.GeoAxesSubplot. Currently: {type(ax)}"
    points = geodf.geometry.values

    # [add_point_location_to_map(point, ax, color="0037ff") for point in points]
    ax.scatter(
        [point.x for point in points],
        [point.y for point in points],
        transform=cartopy.crs.PlateCarree(),
        color=point_colors,
    )

    return ax


import time
import requests

# Цикл проверки и авторизации сети
while True:
    # Проверка подключения к сети
    try:
        r = requests.get('http://172.30.0.36:8000/')
    except:
        print('Подключитесь к сети')
        time.sleep(120)
        continue
    # Проверка авторизации в сети
    try:
        auto = requests.get('https://www.google.ru/?hl=ru')
        if auto.status_code == requests.codes.ok:
            print('Вы авторизированы !')
    # Авторизация в сети
    except:
        print('Отправка POST-запроса')
        data = {
        'user': 'Guest',
        'password': ''
        }
        requests.post('http://172.30.0.36:8000/api/captiveportal/access/logon/0/', data=data)