import math, random, timeit, cv2, time, numpy as np
from funcs import *
from piece import pc
np.warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)

class puzzle:
    def __init__(self, pcs, dims, startPiece=None):
        self.pcs = pcs
        assert dims[0]*dims[1]==len(pcs), f"the given dimensions is not compatible with number of given pieces. len of array is {len(pcs)}"
        self.unplaced = [e for e in pcs]
        self.placed = []
        self.dims = dims
        self.edgepcs = [e for e in self.pcs if sum(e.sideTypes==0) == 1]
        self.cornerpcs = [e for e in self.pcs if sum(e.sideTypes==0) == 2]
        self.solved = False
        self.maxX, self.maxY = 0, 0
        self.state = [[None]*dims[0] for i in range(dims[1])]

        if len(self.cornerpcs) > 0:
            s = self.cornerpcs[0] if startPiece==None else self.pcs[startPiece]
            r = [i for i in range(5) if s.sideTypes[i%4]==0 and s.sideTypes[(i+1)%4]==0]
            self.place(s, (0,0), r[0])

        self.cheat = [1, 3, 3, 3, 0, 0, 1, 0, 3, 2,
                      0, 1, 0, 3, 0, 0, 0, 1, 1, 3,
                      2, 0, 2, 2, 3, 2, 1, 2, 1, 3,
                      2, 2, 0, 0, 3, 1, 0, 0, 0, 1,
                      1, 1, 1, 2, 0, 0, 1, 2, 1, 0,
                      3, 2, 1, 2, 1, 1, 2, 0, 2, 3]

    def place(self, p, pos, r, keep=True, state=None):
        if type(p) == pc:
            p = [p]
            r = [r]
            pos = [pos]
        for i, pz in enumerate(p):
            pz.pos = pos[i]
            pz.rotation = r[i]
            assert not (keep and state!=None), "cannot keep board state from hypothetical state input"
            if keep and state==None:
                self.unplaced.remove(pz)
                self.placed.append(pz)
                self.solved = len(self.unplaced)==0
                self.maxX = pz.pos[0] if self.maxX<pz.pos[0] else self.maxX
                self.maxY = pz.pos[1] if self.maxY<pz.pos[1] else self.maxY
                self.state[pos[i][1]][pos[i][0]] = pz
                st = self.state
            else:
                #st = [[e for e in r] for r in self.state] if type(state)==type(None) else state
                st = self.state if type(state)==type(None) else state
                st[pos[i][1]][pos[i][0]] = pz
        return st

    def getBorders(self, pos, state=None):
        s = state if state!=None else self.state
        x, y = pos[0], pos[1]
        c = [(x,y+1), (x-1,y), (x,y-1), (x+1,y)]
        borders = [None, None, None, None]
        for row in s:
            for p in row:
                if p != None and p.pos in c:
                    w = c.index(p.pos)
                    borders[w] = (p, (p.rotation-w)%4)
        return borders

    def perimeterPositions(self, prio=True, state=None):
        s = state if state!=None else self.state
        borders = [[], [], [], []] if prio else []
        for i, row in enumerate(s):
            for j in range(len(row)):
                if s[i][j] == None:
                    pos = (j, i)
                    m = (np.array(self.getBorders(pos, state=s))!=None).sum()
                    if m > 0:
                        if prio:
                            borders[4-m].append(pos)
                        else:
                            borders.append(pos)
        if prio:
            for i, e in enumerate(borders):
                if len(e) > 0:
                    return 4-i, e
        return borders

    def bestPlacement(self, returnAll=False, maxPcs=3):
        numborder, spots = self.perimeterPositions()
        best = None
        if numborder == 1 and maxPcs >= 2:
            tspots = []
            dspots = self.canFit([(0,0),(1,0)], spots) + self.canFit([(0,0),(0,1)], spots)
            if maxPcs >= 3: tspots = self.canFit([(0,0),(1,0),(2,0)], spots) + self.canFit([(0,0),(0,1),(0,2)], spots)
            if len(dspots) > 0: spots = dspots
            if len(tspots) > 0: spots = tspots
        for spot in spots:
            pc, rot, score = self.bestFit(spot)
            if best==None or score < best[3]:
                best = (pc, spot, rot, score)
        if returnAll:
            return self.bestFit(best[1], returnAll=True)
        return best

    def canFit(self, shape, spots):
        assert (0,0) in shape, "input shape must include (0, 0) and all relative positions to (0,0), if any"
        can = []
        for spot in spots:
            relatives = {(spot[0]+p[0], spot[1]+p[1]) for p in shape}
            if relatives.issubset(set(spots)):
                can.append(list(relatives))
        return can

    def bestFit(self, pos, state=None, returnAll=False, scoreThreshold=25):
        sta = self.state if state==None else state
        best = None
        lol = []
        if type(pos[0]) == tuple and len(pos) == 1:
            pos = pos[0]
        if type(pos)==tuple:
            for pc in self.unplaced:
                for rot in range(4):
                    score = self.evalPlacement(pc, pos, rot, state=sta)
                    if best==None or score < best[2]:
                        best = (pc, rot, score)
                    if returnAll: lol.append((self.pcs.index(pc), pos, rot, score))
        else:
            for p in pos:
                assert type(p) == tuple, f"position argument was not recognized as a tuple or list of tuples"
            compatible = []
            total = 1
            for i in range(len(pos)):
                candidates, all = [], []
                noborders = len([1 for e in self.getBorders(pos[i]) if e != None]) == 0
                for r in range(4):
                    for q in range(len(self.unplaced)):
                        if not noborders:
                            s = self.evalPlacement(self.unplaced[q], pos[i], r, state=sta)
                            if s < scoreThreshold:
                                candidates.append((self.unplaced[q], r))
                        elif self.validEdgePlacement(self.unplaced[q], pos[i], r):
                            all.append((self.unplaced[q], r))
                if len(candidates) == 0:
                    candidates = all
                total *= len(candidates)
                compatible.append(candidates)
            dim = tuple(len(e) for e in compatible)
            print(f"finding best for spots {pos}. search space is of dimensions {dim}: {total} possibilities")
            for i in range(total):
                coord = np.unravel_index(i, dim)
                pzs, rs = [], []
                for j, e in enumerate(coord):
                    pzs.append(compatible[j][e][0])
                    rs.append(compatible[j][e][1])
                score = self.multiPlaceEval(pzs, pos, rs)
                if returnAll: lol.append(([self.pcs.index(e) for e in pzs], pos, rs, score))
                #print(f"{coord}:{[self.pcs.index(x) for x in pzs]}, {rs}, score: {score}")
                if best == None or score < best[2]:
                    best = (pzs, rs, score)
        if returnAll:
            lol.sort(key=lambda x: -x[3])
            return lol
        #assert best[3] < 50, f"No appropriate matches found for position/s {pos}. Best placement is {best}"
        return best

    def multiPlaceEval(self, pzs, pos, rot, state=None, returnAll=False, scoreThreshold=25):
        assert len(pzs)==len(pos)==len(rot)>1, "input lists are not of the same size, or are less than length 2"
        fit = []
        st = state if state!=None else [[e for e in r] for r in self.state]
        for i, pc in enumerate(pzs):
            st = self.place(pc, pos[i], rot[i], state=st, keep=False)
        for i, pc in enumerate(pzs):
            f = self.evalPlacement(pc, pos[i], rot[i], state=st)
            if f>scoreThreshold: return 1e6
            fit.append(f)
        return sum(fit)/len(fit)

    def evalPlacement(self, p, pos, rot, state=None, show=False):
        sta = self.state if state==None else state
        borders = self.getBorders(pos, state=sta)
        scores = []
        ims = []
        for i, b in enumerate(borders):
            if b != None:
                o, s = b
                m = (rot-i+2)%4
                valid = self.validEdgePlacement(p, pos, rot)
                if valid:
                    out = self.evalMatch((p, o), (m, s), show=show)
                    if show:
                        score, im = out
                        scores.append(score)
                        ims.append(im)
                    else:
                        scores.append(out)
                else:
                    return 1e6
        assert len(scores) != 0, "No match evaluation performed: 0 bordering edges."
        s = sum(scores)/len(scores)
        if show:
            t = np.array(ims[0], dtype=np.uint8)
            for i in range(1, len(ims)):
                st, sim = np.shape(t), np.shape(ims[i])
                diff = st[0] - sim[0]
                if diff > 0: ims[i] = np.pad(ims[i], ((0,diff),(0,0),(0,0)), "constant", constant_values=0)
                if diff < 0: t = np.pad(t, ((0,abs(diff)),(0,0),(0,0)), "constant", constant_values=0)
                t = np.concatenate((t, ims[i]), axis=1)
            return s, t
        return s

    def evalMatch(self, pieces, sidenums, ignoreRules=False, show=False, thickness=1, search=True):
        first, second = sidenums
        pc1, pc2 = pieces
        types1, types2 = pc1.sideTypes, pc2.sideTypes
        s1, s2 = pc1.correctedSides[first], pc2.correctedSides[second]
        validsides = True
        d = -1
        if (0 in types1) and (0 in types2):
            validsides = (types1[(first-1)%4]==0 and types2[(second+1)%4]==0) or (types1[(first+1)%4]==0 and types2[(second-1)%4]==0)
        elif (0 in types1) and (not 0 in types2):
            if sum(types1==0)==2:
                validsides = False
            else:
                i = np.where(types1==0)[0]
                validsides = not (first!=((i+2)%4))
        elif (not 0 in types1) and (0 in types2):
            if sum(types2==0)==2:
                validsides = False
            else:
                i = np.where(types2==0)[0]
                validsides = not (second!=((i+2)%4))
        elif pc1.sideTypes[first] == pc2.sideTypes[second] or (types1[first]==0) or (types2[second]==0):
            validsides = False
        else:
            d1, d2 = dist(s1[0], s1[-1]), dist(s2[0], s2[-1])
            d = abs(d1 - d2)
            validsides = d < 30
        validsides = validsides and not ignoreRules
        if not validsides:
            fit = 1e6
        else:
            s1, s2 = pc1.sides[first], pc2.sides[second]
            s1, s2 = s1 - s1[0], np.flipud(s2) - s2[-1]
            offset = math.atan2(s1[-1][1], s1[-1][0]) - math.atan2(s2[-1][1], s2[-1][0])
            s2 = rotate(s2, offset)
            fit = listSim(s1, s2)
            if search:
                stepsize = 1
                steps = 0
                decay = .96
                shifts = np.array([[-1,0], [1,0], [0,-1], [0,1]])
                fits = [listSim(s1-e, s2) for e in shifts]
                shifts = shifts*stepsize
                while fit-.02 > min(fits) and steps < 15:
                    fit = listSim(s1, s2)
                    shifts = shifts*(decay**steps)
                    fits = [listSim(s1-e, s2) for e in shifts]
                    steps += 1
                    s1 = s1-shifts[np.argmin(fits)]
                #print(f"improved score by {ofit-fit} after travelling {dist(s1o[0], s1[0])} pixels in {steps} steps")
        if show:
            if not validsides:
                s1, s2 = s1 - s1[0], s2 - s2[0]

            x, y, w, h = cv2.boundingRect(np.append(s1, s2, axis=0).astype(np.float32))
            origin = [-70+x, -70+y]
            s1, s2 = np.array(s1)-origin, np.array(s2)-origin
            im = np.zeros((h+100, w+100, 3), np.uint8)
            im = cv2.putText(im, str(round(fit, 3)), (round(.5*w), 50), cv2.FONT_HERSHEY_DUPLEX, 1, color=(250, 80, 80), lineType=cv2.LINE_AA)
            im = cv2.polylines(im, np.int32([s1]), False, (50, 250, 50), thickness) # green is s1
            im = cv2.polylines(im, np.int32([s2]), False, (50, 0, 250), thickness) # red is s2
            #(round(.75*(h+100)), round(.75*(w+100)))
            return fit, im
        return fit

    def validEdgePlacement(self, p, pos, rot):
        x, y = pos
        w, h = self.dims
        if x == 0 and p.sideTypes[(rot+1)%4] != 0:
            return False
        if y == 0 and p.sideTypes[rot] != 0:
            return False
        if y == h-1 and p.sideTypes[(rot+2)%4] != 0:
            return False
        if x == w-1 and p.sideTypes[(rot-1)%4] != 0:
            return False
        if sum(p.sideTypes==0) > 0 and not (x in [0, w-1] or y in [0, h-1]):
            return False
        if sum(p.sideTypes==0) == 2 and pos not in [(0,0), (w-1,0), (w-1,h-1), (0,h-1)]:
            return False
        return True

    def showState(self, state=None):
        s = self.state if state==None else state
        for row in s:
            r = bcolors.OKBLUE
            for e in row:
                if e != None:
                    idx = self.pcs.index(e)
                    r += (" "+str(idx)+" " if idx < 10 else str(idx)+" ")
                else:
                    r += " x "
            print(r + bcolors.ENDC)

    def emptyState(self):
        return [[None]*self.dims[0] for i in range(self.dims[1])]

    def show(self, scale=1):
        tw = 0
        for pc in self.placed:
            w, h, d = np.shape(pc.cropped)
            tw = w if w > tw else tw
        pzl = np.ones((tw*(self.maxY+1), tw*(self.maxX+1), 3), dtype=np.uint8)*255

        rot = np.array([[math.cos(1.571), -math.sin(1.571)], [math.sin(1.571), math.cos(1.571)]])
        for pc in self.placed:
            x, y = pc.pos
            padded = np.ones((tw, tw, 3), dtype=np.uint8)*255
            ch, cw, d = np.shape(pc.cropped)
            if ch != cw:
                new = min(ch, cw)
                crop = cv2.resize(pc.cropped, (new, new), interpolation=cv2.INTER_NEAREST)
                padded[:new,:new] = crop
            else:
                padded[:ch,:cw] = pc.cropped

            c1, c2 = pc.croppedCorners[pc.rotation],  pc.croppedCorners[(pc.rotation+1)%4]
            dir = (c1-c2)/dist(c1, c2)
            centerpt = [round((c1[0] + c2[0])/2), round((c1[1] + c2[1])/2)]
            perp = np.dot(dir, rot)
            tri = np.int32([[centerpt-55*dir-55*perp], [centerpt+55*dir-55*perp], [centerpt]])

            i = pc.rotation
            padded = cv2.fillPoly(padded, [tri], color=(250-70*i, 150-50*i, 80*i), lineType=cv2.LINE_AA)
            pzl[y*tw:(y+1)*tw, x*tw:(x+1)*tw] = padded

        return imscale(pzl, scale)


__doc__ = """
    Describes logic of selection algorithm based on Sobol sequences in Sobol space."""

import sobol_seq
import logging

from selection.selection_algorithm_abs import SelectionAlgorithm
from scipy.spatial.distance import euclidean
from itertools import product

class SobolSequence(SelectionAlgorithm):
    def __init__(self, experiment):
        """
        Creates SobolSequence instance that stores information about number of generated points
        :param experiment: the instance of Experiment class
        """

        self.search_space = experiment.description["DomainDescription"]["AllConfigurations"]
        self.dimensionality = len(self.search_space)
        self.numOfGeneratedPoints = 0  # Counter of retrieved points from Sobol sequence.
        self.returned_points = []  # Storing previously returned points.
        self.hypercube_coordinates = []
        self.logger = logging.getLogger(__name__)

        # Need to use floating numbers of indexes for searching distances between target point
        # and other points in hypercube
        for dimension in self.search_space:
            dim_indexes = [float(x) for x in range(len(dimension))]
            self.hypercube_coordinates.append(dim_indexes)

        # Building hypercube
        self.hypercube = list(product(*self.hypercube_coordinates))

    def __generate_sobol_vector(self):
        """
            Generates a next sobol vector in the current search space.
            :return: sobol vector as numpy array.
        """

        # https://github.com/naught101/sobol_seq#usage
        vector, _ = sobol_seq.i4_sobol(self.dimensionality, self.numOfGeneratedPoints + 1)
        self.numOfGeneratedPoints += 1

        return vector

    def __impose_point_to_search_space(self, point):
        """
            Generates sobol sequence of uniformly distributed data points in N dimensional space.
            :param number_of_data_points: int - number of points that needed to be generated in this iteration
            :param skip: int - number of points to skip from the beginning of sequence,
                               because sobol_seq.i4_sobol_generate stateless.
            :return: sobol sequence as numpy array.
            Takes point with coordinates of exact configuration from search space and retrieves real configuration.
        :param point: list.
        :return: list. Configuration from search space.
        """

        imposed_point = []

        for dimension_index, dimension_value in enumerate(point):
            imposed_point.append(self.search_space[int(dimension_index)][int(dimension_value)])

        return imposed_point

    def get_next_configuration(self):
        """
            Will return next data point from initiated Sobol sequence imposed to the search space.
        :return: list - point in current search space.
        """
        # Getting next point from sobol sequence.
        point = self.__generate_sobol_vector()

        # Imposed this point to hypercube dimension sizes.
        point = [len(self.hypercube_coordinates[dimension_index]) * dimension_value for dimension_index, dimension_value
                 in enumerate(point)]

        # Calculate dictionary (keys - distance, values - point) with distances to self.numOfGeneratedPoints
        # nearest points in search space from this point.
        #
        # self.numOfGeneratedPoints + 1(and current point), because in the worst case we will
        # skip(because they was previously returned) this number of points

        distances_dict = {}
        for hypercube_point in self.hypercube:
            if len(distances_dict) < self.numOfGeneratedPoints + 1:
                distances_dict[euclidean(point, hypercube_point)] = hypercube_point
            elif len(distances_dict) == self.numOfGeneratedPoints + 1:
                distances = list(distances_dict.keys())
                distances.sort()
                del(distances_dict[distances.pop()])
                distances_dict[euclidean(point, hypercube_point)] = hypercube_point

        # Picking existing configuration point from hypercube by the smallest distance to generated by Sobol point,
        # if it was not previously picked.
        unique_point = None
        distances = list(distances_dict.keys())
        distances.sort()
        for current_distance in distances:
            candidate = self.__impose_point_to_search_space(distances_dict[current_distance])
            if candidate not in self.returned_points:
                unique_point = candidate
                self.returned_points.append(candidate)
                break
        if not unique_point:
            #   In this point Sobol unable to generate unique configuration as all
            # configurations have been returned at least once.
            unique_point = self.__impose_point_to_search_space(point)
            self.logger.warn("Retrieving not unique configuration point from the Sobol selection algorithm!")
        self.logger.debug("Retrieving new configuration from the Sobol sequence: %s" % str(unique_point))
        return unique_point

    def __disable_point(self, point):
        """
            This method should be used to let Sobol sequence know,
            that some points of search space have been already picked by prediction model.
        :param point: list. Point from search space.
        :return: None
        """
        if point not in self.returned_points:
            self.returned_points.append(point)
            return True
        else:
            self.logger.warn("WARNING! Trying to disable point that have been already retrieved(or disabled).")
            return False

    def disable_configurations(self, configurations):
        """
            This method should be used to forbid points of the Search Space that
            have been already picked as a Model prediction.
        :param configurations: list of configurations. Configurations from search space.
        :return: None
        """
        for p in configurations:
            self.__disable_point(p.get_parameters())

if __name__ == "__main__":
    """
    Add these lines to the beginning of the script to be able to run it stand-alone. (fixes imports):

from os import chdir
from os.path import abspath
from sys import path
chdir('..')
path.append(abspath('.'))
    """
    from time import time
    from random import choice

    def test_for_duplicates(generated_configs):
        # helper function
        for config in generated_configs:
            if generated_configs.count(config) > 1:
                print("Repeated(%s times) configuration(%s) found!" % (generated_configs.count(config), config))
                return True
        return False

    test_space = [[x for x in range(8)],
                  [x / 10 for x in range(8)],
                  [x * 10 for x in range(8)]]  # 3D search space with 512 points in it.

    print("Testing the basic functionality for retrieving an unique points.")
    started = time()
    sobol = SobolSequence(None, test_space)
    generated_configs = [sobol.get_next_configuration() for _ in range(len(list(product(*test_space))))]
    assert test_for_duplicates(generated_configs) is False, 'Got duplicates in normal search space.'
    print("Time to generate all(%s) points in these search space: %s " % (len(generated_configs), time() - started))

    print("Testing basic functionality for retrieving more points that the search space contains. Duplicates appears.")
    sobol = SobolSequence(None, test_space)
    generated_configs = [sobol.get_next_configuration() for _ in range(len(list(product(*test_space))) + 1)]
    assert test_for_duplicates(generated_configs) is True, 'Unique configs with exceeding search space, investigate.'

    print("Testing disabling the same point multiple times and further proper work of the Sobol.")
    sobol = SobolSequence(None, test_space)
    configuration = choice(list(product(*test_space)))
    assert sobol.__disable_point(configuration) is True, "Unable to disable configuration!"
    assert sobol.__disable_point(configuration) is False, "Able to disable same configuration twice!"
    generated_configs = [sobol.get_next_configuration() for _ in range(len(list(product(*test_space))) - 1)]
    assert test_for_duplicates(generated_configs) is False, "Got duplicates in search space with disabled configurations."

    print("Testing the multiple points disabling and further proper work of the Sobol.")
    sobol = SobolSequence(None, test_space)
    for point in list(product(*test_space))[0: int(len(list(product(*test_space)))/3)]:
        sobol.__disable_point(point)
    generated_configs = [sobol.get_next_configuration() for _ in range(int(len(list(product(*test_space)))/3))]
    assert test_for_duplicates(generated_configs) is False, \
        "Got duplicates with multiple (%s) disabled points!" % len(sobol.returned_points)

    print("\nUnit test pass.")


from kivy.properties import ObjectProperty
from kivy.uix.screenmanager import NoTransition, ScreenManager

from widgets import utils
from widgets.homepage.HomePage import HomeScreen
from widgets.Window import Window
from widgets.navdrawer.helpmenu.HelpPage import HelpPage

class WindowManager(ScreenManager):
    nav_drawer = ObjectProperty()

    def __init__(self, **kwargs):
        super(WindowManager, self).__init__(**kwargs)
        # adding all the pages found in the pages json file and creating a new project if none are found

        utils.window_manager = self
        if len(utils.keys) == 1:
            self.add_widget(Window(utils.keys[0], utils.keys[0], utils.keys[0], utils.data[utils.keys[0]]))
        else:
            for window in utils.keys:
                if window == utils.keys[0] and utils.keys[utils.keys.index(window) + 1] is not None:
                    self.add_widget(Window(window, window, utils.keys[utils.keys.index(window) + 1], utils.data[window]))
                elif window == utils.keys[-1] and utils.keys[utils.keys.index(window) - 1] is not None:
                    self.add_widget(Window(window, utils.keys[utils.keys.index(window) - 1], window, utils.data[window]))
                else:
                    self.add_widget(
                        Window(window, utils.keys[utils.keys.index(window) - 1],
                               utils.keys[utils.keys.index(window) + 1],
                               utils.data[window]))
        self.home_screen = HomeScreen(self, "home_select")
        self.add_widget(self.home_screen)
        self.transition = NoTransition()
        self.current = "home_select"

        self.help_screen = HelpPage("help_page")
        self.add_widget(self.help_screen)

    def add_null_window(self, window_data_key):
        # if len(utils.keys) == 1:
        new_win = Window(window_data_key, window_data_key, window_data_key, utils.data[window_data_key], False)
        # else:
        #     new_win = Window(window_data_key, utils.windows[-2], window_data_key[-1], utils.data[window_data_key], False)
        self.add_widget(new_win)
        self.home_screen.home_select_page.add_card(window_data_key)
        utils.update_changes()


"""empty message

Revision ID: b461795a6389
Revises:
Create Date: 2021-09-07 17:18:57.405513

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = 'b461795a6389'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('channels',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('channelName', sa.String(), nullable=True),
    sa.Column('createdTime', sa.DateTime(), nullable=True),
    sa.Column('updatedTime', sa.DateTime(), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('channelName')
    )
    op.create_index(op.f('ix_channels_id'), 'channels', ['id'], unique=False)
    op.create_table('mobiles',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('number', sa.String(), nullable=True),
    sa.Column('createdTime', sa.DateTime(), nullable=True),
    sa.Column('updatedTime', sa.DateTime(), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('number')
    )
    op.create_index(op.f('ix_mobiles_id'), 'mobiles', ['id'], unique=False)
    op.create_table('users',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('first_name', sa.String(), nullable=True),
    sa.Column('last_name', sa.String(), nullable=True),
    sa.Column('age', sa.Integer(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_users_id'), 'users', ['id'], unique=False)
    op.create_table('groups',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('mobile_id', sa.Integer(), nullable=True),
    sa.Column('Channel_id', sa.Integer(), nullable=True),
    sa.Column('createdTime', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['Channel_id'], ['channels.id'], ),
    sa.ForeignKeyConstraint(['mobile_id'], ['mobiles.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('messages',
    sa.Column('messageID', postgresql.UUID(as_uuid=True), nullable=False),
    sa.Column('message', sa.String(), nullable=True),
    sa.Column('createdTime', sa.DateTime(), nullable=True),
    sa.Column('updatedTime', sa.DateTime(), nullable=True),
    sa.Column('channel_id', sa.Integer(), nullable=True),
    sa.Column('mobile_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['channel_id'], ['channels.id'], ),
    sa.ForeignKeyConstraint(['mobile_id'], ['mobiles.id'], ),
    sa.PrimaryKeyConstraint('messageID')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('messages')
    op.drop_table('groups')
    op.drop_index(op.f('ix_users_id'), table_name='users')
    op.drop_table('users')
    op.drop_index(op.f('ix_mobiles_id'), table_name='mobiles')
    op.drop_table('mobiles')
    op.drop_index(op.f('ix_channels_id'), table_name='channels')
    op.drop_table('channels')
    # ### end Alembic commands ###


# coding: utf-8
import argparse
from pyquery import PyQuery as pq

def getElementBySelector(URL, selector):
    pqobj = pq(URL)
    parseUrl = pqobj(selector);

    # if default selector then return page content
    if selector == "html":
        result = parseUrl.outerHtml()
    #else return list of found elements
    else :
        result = [i.text() for i in parseUrl.items()]
    return result

def isVowel(letter):
    vowels = ("a",  "e",  "i",  "o",  "u",  "y",                        #english alphabet
              "а",  "е",  "ё",  "и",  "о",  "у",  "ы",  "э", "ю",  "я") #russian alphabet
    return letter.lower() in vowels;

def isLetter(letter):
    letter_code = ord(letter.lower())

    #97-122           : english alphabet letters
    english_letter = 97 <= letter_code <= 122

    #1072-1103 и 1105 : russian alphabet letters
    russian_letter = 1072 <= letter_code <= 1105 or letter_code == 1105

    if english_letter or russian_letter:
        return True
    else:
        return False

def mySortFunction(data):
    if type(data) != str:
        vowels = []
        consonant = []
        other = []

        #divide by letters
        for i in data:
            if isVowel(i[0]):
                vowels.append(i)
            elif isLetter(i[0]):
                consonant.append(i)
            else:
                other.append(i)

        vowels.sort(key=lambda inputString: inputString[0])
        consonant.sort(key=lambda inputString: inputString[0])
        other.sort(key=lambda inputString: inputString[0])

        result = [i for i in vowels]
        for i in consonant:
            result.append(i)
        for i in other:
            result.append(i)

        return result

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Parser for any url using css-selector.")
    parser.add_argument("URL", type=str, help="url to parse")

    parser.add_argument("-c", "--css-selector", type=str, default="html",
                    help="selector to get elements. By default selector = 'html'")

    parser.add_argument("-s", "--sorting", action="store_true",
                    help="include the option to sord output data. By default=False")
    args = parser.parse_args()

    result = getElementBySelector(args.URL,  args.css_selector)

    if args.sorting:
        result = mySortFunction(result)

    # танцы с бубном над codepage
    # http://trust-host.ru/support-host/site/175-kodirovka-cp1251-na-sajte.html
    #result = result.encode('iso-8859-1').decode("cp1251").encode("utf-8").decode("utf-8")

    for i in result:
        print(i)


#!/usr/bin/python

import urllib2
import json
import sys
import os
import argparse
from collections import defaultdict

def read_auth(filename):
    with open(filename, 'rb') as auth_file:
        key = auth_file.readline()[:-1]

    return key

parser = argparse.ArgumentParser("Retrieves traceroute results")
parser.add_argument('--datadir', required=True, help="directory to save output")
parser.add_argument('--msm', required=False, type=int, help="Fetch specific measurement id")
args = parser.parse_args()

if not os.path.exists(args.datadir):
    print("Data directory %s must exists!" % args.datadir)
    sys.exit(1)

atlas_read_api_key = 'read-key.txt'
authkey = read_auth(atlas_read_api_key)
if authkey is None:
    print("Auth file %s not found, aborting" % atlas_read_api_key)
    sys.exit(1)

if not args.msm:
    with open(os.path.join(args.datadir, 'measurements.json')) as msm_file:
        msm_data = json.load(msm_file)
else:
    msm_data = {'cc': [args.msm]}

result_list = defaultdict(list)
for cc, msm_list in msm_data.iteritems():
    for msm in msm_list:
        print("Fetching results for measurement %s" % msm)
        api_url = "https://atlas.ripe.net/api/v1/measurement/{}/result/?key={}".format(msm, authkey)
        request = urllib2.Request(api_url)
        request.add_header("Accept", "application/json")
        try:
            conn = urllib2.urlopen(request)
            msm_data = json.load(conn)
            for result in msm_data:
                result_list[cc].append(result)
            conn.close()
        except urllib2.HTTPError as e:
            print >> sys.stderr, ("Fatal error: %s" % e.read())
            raise

with open(os.path.join(args.datadir, 'results.json'), 'wb') as res_file:
    json.dump(result_list, res_file)


import urllib
import http.cookiejar
import re
import bz2

cookiejar = http.cookiejar.CookieJar() #声明一个CookieJar对象实例来保存cookie
handler = urllib.request.HTTPCookieProcessor(cookiejar) #利用HTTPCookieProcessor对象来创建cookie处理器
opener = urllib.request.build_opener(handler,urllib.request.HTTPHandler) #通过handler来构建opener
urllib.request.install_opener(opener) #安装opener作为urlopen()使用的全局URL opener，即以后调用urlopen()时都会使用安装的opener对象。

num = '12345'
url_prefix = 'http://www.pythonchallenge.com/pc/def/linkedlist.php?busynothing='
infojar = []
while True:
    file = urllib.request.urlopen(url_prefix+num)
    data = file.read().decode('UTF-8')
    print(data)
    infojar.append(list(cookiejar)[0].value)
    file.close()
    if re.findall('([0-9]+)',data):
        num = re.findall('([0-9]+)',data)[-1]
    else:
        break

unquote_info = urllib.parse.unquote_to_bytes(''.join(infojar).replace('+', '%20'))
print(bz2.decompress(unquote_info).decode('utf-8'))


# coding=utf8
from setuptools import setup
__author__ = 'hellflame'

setup(
    name='paramSeeker',
    version="0.3.2",
    keywords=('param', 'parameter', 'terminal handler'),
    description="Terminal parameter retrive then 执行以及参数转发，开发实体内容",
    license='Apache License',
    author='hellflame',
    author_email='hellflamedly@gmail.com',
    url="https://github.com/hellflame/paramSeeker",
    packages=[
        'paramSeeker'
    ],
    platforms="any",
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: Apache Software License',
        'Topic :: Software Development :: Libraries',
        'Topic :: Software Development :: Libraries :: Python Modules',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.4',
        "Operating System :: OS Independent"
    ],
    entry_points={
        'console_scripts': [
            'seeker=paramSeeker.example:test_env'
        ]
    }
)


import random
import time

from dagster import (
    Field,
    InputDefinition,
    ModeDefinition,
    Output,
    OutputDefinition,
    pipeline,
    solid,
)
from dagster.core.definitions.executor import default_executors

def get_executor_defs():
    try:
        from dagster_dask import dask_executor

        return default_executors + [dask_executor]
    except ImportError:
        return default_executors

@solid(
    input_defs=[InputDefinition('chase_duration', int)],
    output_defs=[OutputDefinition(int, 'total')],
    config_schema={
        'chase_size': Field(
            int,
            default_value=100000,
            is_required=False,
            description='How big should the pointer chase array be?',
        )
    },
)
def hammer(context, chase_duration):
    '''what better way to do a lot of gnarly work than to pointer chase?'''
    ptr_length = context.solid_config['chase_size']

    data = list(range(0, ptr_length))
    random.shuffle(data)

    curr = random.randint(0, ptr_length)
    # and away we go
    start_time = time.time()
    while (time.time() - start_time) < chase_duration:
        curr = data[curr]

    context.log.info('Hammered - start %d end %d' % (start_time, time.time()))
    return chase_duration

@solid(
    config_schema=Field(int, is_required=False, default_value=1),
    output_defs=[
        OutputDefinition(int, 'out_1'),
        OutputDefinition(int, 'out_2'),
        OutputDefinition(int, 'out_3'),
        OutputDefinition(int, 'out_4'),
    ],
)
def chase_giver(context):
    chase_duration = context.solid_config

    yield Output(chase_duration, 'out_1')
    yield Output(chase_duration, 'out_2')
    yield Output(chase_duration, 'out_3')
    yield Output(chase_duration, 'out_4')

@solid(
    input_defs=[
        InputDefinition('in_1', int),
        InputDefinition('in_2', int),
        InputDefinition('in_3', int),
        InputDefinition('in_4', int),
    ],
    output_defs=[OutputDefinition(int)],
)
def reducer(_, in_1, in_2, in_3, in_4):
    return in_1 + in_2 + in_3 + in_4

@pipeline(
    # Needed for Dask tests which use this pipeline
    mode_defs=[ModeDefinition(executor_defs=get_executor_defs())]
)
def hammer_pipeline():

    out_1, out_2, out_3, out_4 = chase_giver()
    return reducer(
        in_1=hammer(chase_duration=out_1),
        in_2=hammer(chase_duration=out_2),
        in_3=hammer(chase_duration=out_3),
        in_4=hammer(chase_duration=out_4),
    )


"""
Concepts: you can decorate methods. And classes!

Same thing but for classes. This introduces a new wrinkle when decorating methods.

The problem is that our decorator is invoked as each method is defined - we get
the function that results from the definition.

But to store that function function we're going to need to have access to the
containing class to get to the `ENDPOINTS` class variable... and the class
hasn't been defined when our decorator runs!

We need to defer actually collecting the methods until after the class has been
defined - when it can be done by a class decorator!

Update the `endpoint` decorator to mark decorated methods and then update the
`collect` decorator to loop through the attributes of the class finding the
methods we marked and storing them in the `RemoteAPI.ENDPOINTS` dict.

Hint: help(vars). Also - functions are objects like anything else, open by
default so its totally fine to say `function._my_mark = 1`.

"""
import argparse

def endpoint(f):
    return f

def collect(klass):
    return klass

@collect
class RemoteAPI:
    ENDPOINTS = {}

    def __init__(self, secrets=None):
        # Presumably load some secrets from a file so we can authenticate our client
        pass

    @endpoint
    def sessions(self, event_id=None):
        """Returns the session ids for the event."""
        return [1, 2, 3]

    @endpoint
    def events(self):
        """Returns the events to which you have access"""
        return [2717]

if __name__ == "__main__":
    # Setup CLI options
    parser = argparse.ArgumentParser()
    # Add a subcommand for every endpoint we've implemented
    subparsers = parser.add_subparsers(
        title="Endpoints",
        help="The following endpoints are supported:",
        dest="subcommand",
    )
    client = RemoteAPI()
    # Using inspect to add flags for endpoint arguments ommitted for brevity
    for (name, func) in client.ENDPOINTS.items():
        sub = subparsers.add_parser(name, help=func.__doc__)

    # Pick a subcommand to run
    args = parser.parse_args()
    if not args.subcommand:
        parser.exit("Please specify a subcommand")
    function = client.ENDPOINTS[args.subcommand]

    print(function(client))  # Passing instance of RemoteAPI to self. Do you know why?


'''
Created on May 31, 2016

@author: Peter Hillyard
'''

# This script is used to plot a link using all channels
import numpy as np
import matplotlib.pyplot as plt

fname = 'data/rss_data/pizza_house/rss_2016_05_18_epatch_peter_walk_all.txt'
N = 20
C = 4
L = N*(N-1)*C
link_database = np.zeros((L,4))
link_idx = 0

for ch in range(C):
    for tx in range(N):
        for rx in range(N):
            if tx == rx:
                continue
            link_database[link_idx,0] = link_idx
            link_database[link_idx,1] = tx
            link_database[link_idx,2] = rx
            link_database[link_idx,3] = ch
            link_idx += 1

plot_link=None
plot_pair = [1,2]

if plot_link is not None:
    link_to_plot = int(plot_link)
elif plot_pair is not None:
    tmp = (link_database[:,1] == plot_pair[0]-1) & (link_database[:,2] == plot_pair[1]-1)
    link_to_plot = link_database[tmp,0].astype('int')

timestamp_vec = []
rss_vec = []

with open(fname,'r') as f:
    for line in f:
        split_line = line.split(' ')

        timestamp_vec.append(float(split_line.pop()))
        rss = np.array([float(ii) for ii in split_line])

        rss_vec.append(rss[link_to_plot].tolist())

rss_vec = np.array(rss_vec)
rss_vec[rss_vec == 127] = np.nan
timestamp_vec = np.array(timestamp_vec)-timestamp_vec[0]

plt.plot(timestamp_vec,rss_vec)
plt.show()


import argparse
import json
import os

import numpy as np
import tensorflow as tf

# Read parameters from Valohai
# This enables Valohai to version your parameters,
# run quick iterations and do hyperparameter optimization

parser = argparse.ArgumentParser()
parser.add_argument('--epochs', type=int, default=5)
parser.add_argument('--learning_rate', type=float, default=0.001)
args = parser.parse_args()

# Read input files from Valohai inputs directory
# This enables Valohai to version your training data
# and cache the data for quick experimentation

input_path = os.getenv('VH_INPUTS_DIR', './inputs')
f = os.path.join(input_path, 'preprocessed_mnist/preprocessed_mnist.npz')

with np.load(f, allow_pickle=True) as f:
    x_train, y_train = f['x_train'], f['y_train']
    x_test, y_test = f['x_test'], f['y_test']

model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(input_shape=(28, 28)),
  tf.keras.layers.Dense(128, activation='relu'),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(10, activation='softmax')
])

model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=args.learning_rate),
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# Print metrics out as JSON
# This enables Valohai to version your metadata
# and for you to use it to compare experiments

def log(epoch, logs):
    print()
    print(json.dumps({
        'epoch': epoch,
        'loss': str(logs['loss']),
        'acc': str(logs['acc']),
    }))

cb = tf.keras.callbacks.LambdaCallback(on_epoch_end=log)

model.fit(x_train, y_train, epochs=args.epochs, callbacks=[cb])

# Evaluate the model and print out the test metrics as JSON

test_loss, test_acc = model.evaluate(x_test,  y_test, verbose=2)
print(json.dumps({
    'test_loss': str(test_loss),
    'test_acc': str(test_acc),
}))

# Write output files to Valohai outputs directory
# This enables Valohai to version your data
# and upload output it to the default data store

path = os.getenv('VH_OUTPUTS_DIR', './outputs')
model.save(os.path.join(path, 'model.h5'))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Nov  1 20:09:00 2016

@author: mac
"""

import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
from sklearn.multiclass import OneVsRestClassifier, OneVsOneClassifier
from sklearn.svm import LinearSVC
from sklearn import metrics, tree, cross_validation
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import RandomizedLogisticRegression
import glob
import codecs

data = {"train": {"pos": [], "neg": []},
        "test": {"pos": [], "neg": []}}

txt_types = [("train", "neg"), ("train", "pos"), ("test", "neg"), ("test", "pos")]

# with open should add encoding='utf-8', errors='ignore'
#otherwise failed "'ascii' codec can't decode byte 0xc3 in position 404: ordinal not in range(128)"
for t in txt_types:
    for txt_file in glob.glob("aclImdb/" + t[0] + "/" + t[1] + "/*.txt"):
        with codecs.open(txt_file, "r",encoding='utf-8', errors='ignore') as f:
            text = f.read()
        data[t[0]][t[1]].append(text)
#list(data["train"]["neg"])[0]
# get training + test data

import numpy as np
X_train = data["train"]["pos"] + data["train"]["neg"]
y_train = np.append(np.ones(len(data["train"]["pos"])), np.zeros(len(data["train"]["neg"])))

X_test = data["test"]["pos"] + data["test"]["neg"]
y_test = np.append(np.ones(len(data["test"]["pos"])), np.zeros(len(data["test"]["neg"])))
print(len(X_train), len(y_train))
print(len(X_test), len(y_test))

#tfidf transformation is automately employed in Pipeline
## tfidf = TfidfVectorizer()
## tfidf.fit_transform(X_train)

# build a pipeline - SVC
from sklearn.pipeline import Pipeline
text_clf = Pipeline([('vect', CountVectorizer(ngram_range=(1, 2))),
                    ('tfidf', TfidfTransformer()),
                    ('clf', OneVsRestClassifier(LinearSVC(random_state=0)))
                     ])

# fit using pipeline
clf = text_clf.fit(X_train, y_train)

# predict
predicted = clf.predict(X_test)
clf.score(X_test, y_test)

# print metrics
print(metrics.classification_report(y_test, predicted))

#scores = cross_validation.cross_val_score(text_clf, X_train + X_test, np.append(y_train, y_test), cv=5)

#First we have to one-hot encode the text, but let's limit the features to the most common 20,000 words.
from collections import Counter

max_features = 20000
all_words = []

for text in X_train + X_test:
    all_words.extend(text.split())
unique_words_ordered = [x[0] for x in Counter(all_words).most_common()] #x[1] times
word_ids = {}
rev_word_ids = {}
for i, x in enumerate(unique_words_ordered[:max_features-1]):
    word_ids[x] = i + 1  # so we can pad with 0s
    rev_word_ids[i + 1] = x

X_train_one_hot = []
for text in X_train:
    t_ids = [word_ids[x] for x in text.split() if x in word_ids]
    X_train_one_hot.append(t_ids)

X_test_one_hot = []
for text in X_test:
    t_ids = [word_ids[x] for x in text.split() if x in word_ids]
    X_test_one_hot.append(t_ids)

#Now we can use Keras, a popular Theano wrapper, to quickly build an NN classifier.
from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.preprocessing import sequence
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Embedding
from keras.layers import LSTM, SimpleRNN, GRU

maxlen = 80  # cut texts after this number of words (among top max_features most common words)
batch_size = 32

print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train_one_hot, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test_one_hot, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)

print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128, dropout=0.2))
model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2))  # try using a GRU instead, for fun
model.add(Dense(1))
model.add(Activation('sigmoid'))

# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

print('Train...')
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=15,
          validation_data=(X_test, y_test))
score, acc = model.evaluate(X_test, y_test,
                            batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)


from scipy import stats
from enum import Enum
import numpy as np
from collections import OrderedDict
from stats.Distributions import stats as mystats
from stats.Regressions.QuasilinearModels import *
from stats.Regressions.LinearRegression import LinearRegression
from stats.Regressions.AbstractRegression import AbstractRegression

try:
	from PyQt5 import QtCore
	_ = QtCore.QCoreApplication.translate
except:
	_ = lambda cont, text: text

MODELS = {
	2:	SqrtModel,
	3:	HyperbolicModel,
	4:	InvLinearModel,
	5:	XInvLinearModel,
	6:	LnModel,
	7:	PowModel,
	8:	ExpModel,
	9:	CubicModel,
	10:	ExpInvModel,
	11: InvHyperbolicModel,
	12: InvExpModel
}

class QLRM(Enum):
	SqrtModel = 2
	HyperbolicModel = 3
	InvLinearModel = 4
	XInvLinearModel = 5
	LnModel = 6
	PowModel = 7
	ExpModel = 8
	CubicModel = 9
	ExpInvModel = 10
	InvHyperbolicModel = 11
	InvExpModel = 12

class QuasilinearRegression(AbstractRegression):
	def __init__(self, model=None):
		self._a = 0
		self._b = 0
		self._model = model
		if isinstance(model, QLRM):
			self._model = MODELS[model.value]
		elif isinstance(model, int):
			self._model = MODELS[model]

	def _init_linear_model(self, model):
		regr = LinearRegression()
		if isinstance(model, QLRM):
			model = MODELS[model.value]
		elif isinstance(model, int):
			model = MODELS[model]
		if model == XInvLinearModel:
			psi = model.psi(self._y, self._X)
		else:
			psi = model.psi(self._y)
		phi = model.phi(self._X)
		regr.fit(phi, psi, 1 - self._gamma)
		return regr

	def _find_model(self):
		linregressions = dict()
		for i in MODELS.keys():
			linregressions[i] = self._init_linear_model(MODELS[i])
		kv = 2
		SSR = linregressions[2].SSR()
		for i in linregressions.keys():
			SSR1 = linregressions[i].SSR()
			if SSR1 < SSR:
				kv = i
				SSR = SSR1
		self._model = MODELS[kv]

	def _fit(self, X, y, alpha=0.05):
		a = 0
		b = 0
		self._X = X
		self._y = y
		self._gamma = 1 - alpha
		if self._model is None:
			self._find_model()
		phi = self._model.phi(X)
		args = [y]
		if self._model == XInvLinearModel:
			args.append(X)
		psi = self._model.psi(*args)
		linreg = LinearRegression()
		linreg.fit(phi, psi, 1 - self._gamma)
		B, A = linreg.get_params()
		self._a = A
		self._b = B
		return self

	def _get_psi(self, model):
		y = self._y
		X = self._X
		args = [y]
		if model == XInvLinearModel:
			args.append(X)
		psi = model.psi(*args)
		return psi

	def _get_params(self):
		return (self._a, self._b)

	def _get_paramsdict(self):
		X = self._X
		y = self._y
		phi = self._model.phi(X)
		args = [y]
		if self._model == XInvLinearModel:
			args.append(X)
		psi = self._model.psi(*args)
		linreg = LinearRegression()
		linreg.fit(phi, psi, 1 - self._gamma)
		tests = linreg.get_paramsdict()
		return tests

	def _predict(self, X):
		N = X.shape[0]
		phi = self._model.phi(X)
		A = self._a
		B = self._b
		y = A + B * phi
		if self._model in [MODELS[7], MODELS[8], MODELS[10]]:
			y = np.exp(y)
		elif self._model in [MODELS[4], MODELS[11], MODELS[12]]:
			y = 1 / y
		elif self._model in [MODELS[5]]:
			y = y / X
			y = 1 / y
		elif self._model in [MODELS[2]]:
			y = np.sqrt(y)
		return y

	def _plot(self, ax):
		self.plotRegressionLine(ax)
		#self.plotConfidentInterval(ax)
		#self.plotTolerantBounds(ax)

	def _plotRegressionLine(self, ax=None):
		X = np.linspace(self._X.min(), self._X.max(), self._X.shape[0])
		y = self.predict(X)
		label = _("QuasilinearRegression", "Quasilinear regression")
		ax.plot(X, y, "g-", lw=3, label=label)

	def _plotConfidentInterval(self, ax):
		X = np.linspace(self._X.min(), self._X.max(), self._X.shape[0])
		y = self.predict(X)
		intercept = self._a
		slope = self._b
		N = self._X.shape[0]
		phi = self._model.phi(X)
		if self._model == XInvLinearModel:
			psi = self._model.psi(y, X)
		else:
			psi = self._model.psi(y)
		t = stats.t.ppf(0.5 - self._gamma / 2, N - 2)
		x0, Syx0 = mystats.var_y_x(phi, psi, intercept, slope)
		add = t * Syx0
		ax.plot(X, y + add, "k-", lw=2, label=_("2D", "Confidential intervals"))
		ax.plot(X, y - add, "k-", lw=2)

	def _plotTolerantBounds(self, ax=None):
		X = np.linspace(self._X.min(), self._X.max(), self._X.shape[0])
		N = X.shape[0]
		y = self.predict(X)
		t = stats.t.ppf(0.5 - self._gamma / 2, N - 2)
		SSR = np.sqrt(self.SSR())
		addp = SSR * t
		print(addp)
		print(SSR)
		ax.plot(X, y - addp, 'm-', lw=3, label=_("2D", "Tolerant bounds"))
		ax.plot(X, y + addp, 'm-', lw=3)

	def _SSR(self):
		X = self._X
		y = self._y
		yx = self.predict(X)
		N = X.shape[0]
		SSR = np.sum(yx - y) / (N - 2)
		return np.abs(SSR)

	@staticmethod
	def rvs(model, **kwargs):
		if isinstance(model, QLRM):
			model = model.value
		if isinstance(model, int):
			model = MODELS[model]
		N = kwargs["N"]
		a = kwargs["a"]
		b = kwargs["b"]
		xmin = kwargs["xmin"]
		xmax = kwargs["xmax"]
		eps = stats.norm.rvs(loc=kwargs["o"], scale=kwargs["sigm"], size=N)
		x = np.random.uniform(xmin, xmax, N)
		y = model.y(x, a, b)
		y += eps
		return np.array([x, y]).T

if __name__ == '__main__':
	import seaborn as sns
	kwargs = {
		"N": 500,
		"a": 1,
		"b": 1,
		"o": 0,
		"sigm": 0.0001,
		"xmax": 10,
		"xmin": 0
	}
	model = ExpModel
	x, y = QuasilinearRegression.rvs(model, **kwargs).T
	regr = QuasilinearRegression()
	regr.fit(x, y)
	print(regr._model)
	print(regr.SSR())


from django.conf.urls import patterns, include, url
from django.contrib import admin
from apps.views import UserDetailView

admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'SistemaDiscusiones.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),
    url(r'^', include('apps.home.urls')),
    # url(r'^', include('apps.users.urls', namespace='users')),

    # Python social auth
    url('', include('social.apps.django_app.urls',namespace='social')),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^usuario/(?P<slug>[-\w]+)/', UserDetailView.as_view(), name='user_detail'),
)


import pandas as pd
from files.dictionaries import vertMarkets, locSkills
from common_functions import batches
import salesforce_login
import warnings
warnings.filterwarnings('ignore')
import datetime
from tqdm import tqdm
from simple_salesforce import SalesforceMalformedRequest
import time
import datetime

now = '-'.join('_'.join(str(datetime.datetime.now()).split('.')[0].split()).split(':'))

sf = salesforce_login.bsna()

def realign(territory):

    territoryId = sf.query(f"SELECT Id FROM Territory__c WHERE Name = '{territory}'")['records'][0]['Id']

    skillSets = sf.query_all(f"SELECT OwnerId,Skill_Set__c \
                               FROM Territory_SkillSet_Mapping__c WHERE Territory__r.Id = '{territoryId}' AND SkillSet_Object__c LIKE '%Account Location%'")

    skillDict = {locSkills[x['Skill_Set__c']]:x['OwnerId'] for x in skillSets['records']}

    zips = sf.query_all(f"SELECT Postal_Code__c FROM Postal_Code_Mapping__c WHERE Territory__c = '{territoryId}'")
    zips = [x['Postal_Code__c'] for x in zips['records']]
    zipQuery = batches.likeBatch('Postal_Code__c', zips, 5000)

    def getLocations(zipQuery):

        loc_frames = []

        for query in zipQuery:

            security_locations = sf.query_all(f"SELECT Associated_Location__r.Security_Owner__c,Associated_Location__r.Vertical_Market__c, \
                                                Associated_Location__r.Vertical_Sub_Market__c, Associated_Location__c,Security_Business_ID__c \
                                                FROM Associated_ERP_Account__c \
                                                WHERE ({query}) AND (Associated_Location__r.Security_Owner__c != Null)")
            security_locations = pd.DataFrame.from_records(security_locations['records'])
            loc_frames.append(security_locations)

        security_locations = pd.concat(loc_frames)

        return security_locations

    if len(zipQuery) == 0:
        return
    else:

        security_commercial = getLocations(zipQuery).groupby('Associated_Location__c').\
        filter(lambda x : all(x['Security_Business_ID__c'].str.contains('Commercial'))).drop_duplicates('Associated_Location__c')

        security_commercial = pd.concat([security_commercial.drop(['Associated_Location__r'], axis=1),\
        security_commercial['Associated_Location__r'].apply(pd.Series)], axis=1)

        def wrong_owner(locations):

            banking = locations[locations['Vertical_Market__c'].isin(vertMarkets[0])]
            banking.Security_Owner__c = skillDict['Security_Banking_Skill']

            education = locations[locations['Vertical_Market__c'].isin(vertMarkets[1])]
            education.Security_Owner__c = skillDict['Security_Education_Skill']

            slg = locations[locations['Vertical_Market__c'].isin(vertMarkets[2])]
            slg.Security_Owner__c = skillDict['Security_SLG_Skill']

            territory = locations[locations['Vertical_Market__c'].isin(vertMarkets[3])==False]
            territory.Security_Owner__c = skillDict['Security_TerritoryRep_Skill']

            wrong_owner = pd.concat([banking, education, slg, territory])[['Associated_Location__c','Security_Owner__c']]

            return wrong_owner

        updates = wrong_owner(security_commercial)
        updates = list(zip(updates['Associated_Location__c'], updates['Security_Owner__c']))

        return updates


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('mathtutor', '0006_auto_20150929_0722'),
    ]

    operations = [
        migrations.AddField(
            model_name='quiz',
            name='site',
            field=models.CharField(default=b'tutor', max_length=16, verbose_name=b'Website'),
        ),
    ]


#%%

gamma=1
p=0.4
numStates=100
reward=[0 for _ in range(101)]
reward[100]=1

theta=0.00000001

import matplotlib.pyplot as plt
value=[0 for _ in range(101)]
policy=[0 for _ in range(101)]

def reinforcement_learning():
    delta=1
    while delta>theta:
        delta=0
        for i in range(1,numStates):
            oldValue=value[i]
            bellmanequation(i)
            diff=abs(oldValue-value[i])
            delta=max(delta,diff)
    plt.plot(value)
    plt.xlabel('Capital')
    plt.ylabel('Value Estimates')
    plt.title('Value function')

def bellmanequation(num):
    optimalvalue=0

    for bet in range(0,min(num,100-num)+1):
        win=num+bet
        loss=num-bet

        sum=p*(reward[win]+gamma*value[win])+(1-p)*(reward[loss]+gamma*value[loss])

        if sum>optimalvalue:
            optimalvalue=sum
            value[num]=sum
            policy[num]=bet

reinforcement_learning()


import pytest
import torch
from torch.nn import Sequential, Linear, ReLU, Dropout
import torch.nn.functional as F
from torch_geometric.nn import (GCNConv, GATConv, GNNExplainer,
                                global_add_pool, MessagePassing)

class GCN(torch.nn.Module):
    def __init__(self):
        super(GCN, self).__init__()
        self.conv1 = GCNConv(3, 16)
        self.conv2 = GCNConv(16, 7)

    def forward(self, x, edge_index):
        x = self.conv1(x, edge_index)
        x = F.relu(x)
        x = self.conv2(x, edge_index)
        return x.log_softmax(dim=1)

class GAT(torch.nn.Module):
    def __init__(self):
        super(GAT, self).__init__()
        self.conv1 = GATConv(3, 16, heads=2, concat=True)
        self.conv2 = GATConv(2 * 16, 7, heads=2, concat=False)

    def forward(self, x, edge_index):
        x = self.conv1(x, edge_index)
        x = F.relu(x)
        x = self.conv2(x, edge_index)
        return x.log_softmax(dim=1)

@pytest.mark.parametrize('model', [GCN(), GAT()])
def test_gnn_explainer(model):
    explainer = GNNExplainer(model, log=False)
    assert explainer.__repr__() == 'GNNExplainer()'

    x = torch.randn(8, 3)
    edge_index = torch.tensor([[0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7],
                               [1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6]])

    node_feat_mask, edge_mask = explainer.explain_node(2, x, edge_index)
    assert node_feat_mask.size() == (x.size(1), )
    assert node_feat_mask.min() >= 0 and node_feat_mask.max() <= 1
    assert edge_mask.size() == (edge_index.size(1), )
    assert edge_mask.min() >= 0 and edge_mask.max() <= 1

@pytest.mark.parametrize('model', [GCN(), GAT()])
def test_to_log_prob(model):
    raw_to_log = GNNExplainer(model, return_type='raw').__to_log_prob__
    prob_to_log = GNNExplainer(model, return_type='prob').__to_log_prob__
    log_to_log = GNNExplainer(model, return_type='log_prob').__to_log_prob__

    raw = torch.tensor([[1, 3.2, 6.1], [9, 9, 0.1]])
    prob = raw.softmax(dim=-1)
    log_prob = raw.log_softmax(dim=-1)

    assert torch.allclose(raw_to_log(raw), prob_to_log(prob))
    assert torch.allclose(prob_to_log(prob), log_to_log(log_prob))

def assert_edgemask_clear(model):
    for layer in model.modules():
        if isinstance(layer, MessagePassing):
            assert ~layer.__explain__
            assert layer.__edge_mask__ is None

class Net(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = GCNConv(3, 16)
        self.conv2 = GCNConv(16, 16)
        self.fc1 = Sequential(Linear(16, 16), ReLU(), Dropout(0.2),
                              Linear(16, 7))

    def forward(self, x, edge_index, batch, get_embedding=False):
        x = self.conv1(x, edge_index)
        x = F.relu(x)
        x = self.conv2(x, edge_index)
        if get_embedding:
            return x
        x = global_add_pool(x, batch)
        x = self.fc1(x)
        return x.log_softmax(dim=1)

@pytest.mark.parametrize('model', [Net()])
def test_graph_explainer(model):
    x = torch.randn(8, 3)
    edge_index = torch.tensor([[0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7],
                               [1, 0, 2, 1, 3, 2, 5, 4, 6, 5, 7, 6]])

    explainer = GNNExplainer(model, log=False)

    node_feat_mask, edge_mask = explainer.explain_graph(x, edge_index)
    assert_edgemask_clear(model)
    _, _ = explainer.visualize_subgraph(-1, edge_index, edge_mask,
                                        y=torch.tensor(2), threshold=0.8)
    assert node_feat_mask.size() == (x.size(1), )
    assert node_feat_mask.min() >= 0 and node_feat_mask.max() <= 1
    assert edge_mask.shape[0] == edge_index.shape[1]
    assert edge_mask.max() <= 1 and edge_mask.min() >= 0


import numpy as np
from numpy.testing import assert_allclose, assert_array_almost_equal
import pytest
import torch

from openff.utilities import requires_package
from openff.nagl.nn.activation import ActivationFunction
from openff.nagl.nn.gcn import SAGEConvStack
from openff.nagl.nn.gcn._sage import SAGEConv

try:
    import dgl

    _BASE_SAGECONV_CLASS = dgl.nn.pytorch.SAGEConv
except ImportError:
    _BASE_SAGECONV_CLASS = SAGEConv

class TestDGLSAGEConvStack:
    def test_default_with_layers(self):
        stack = SAGEConvStack.with_layers(
            n_input_features=1,
            hidden_feature_sizes=[2, 3],
        )
        stack.reset_parameters()

        assert len(stack) == 2
        assert all(isinstance(layer, _BASE_SAGECONV_CLASS) for layer in stack)

        first, second = stack
        assert np.isclose(first.feat_drop.p, 0.0)
        assert first.fc_self.in_features == 1
        assert first.fc_self.out_features == 2

        assert np.isclose(second.feat_drop.p, 0.0)
        assert second.fc_self.in_features == 2
        assert second.fc_self.out_features == 3

    def test_with_layers_inputs(self):
        stack = SAGEConvStack.with_layers(
            n_input_features=2,
            hidden_feature_sizes=[3],
            layer_activation_functions=[ActivationFunction.LeakyReLU],
            layer_dropout=[0.5],
            layer_aggregator_types=["lstm"],
        )

        assert len(stack) == 1
        assert all(isinstance(layer, _BASE_SAGECONV_CLASS) for layer in stack)

        layer = stack[0]
        assert np.isclose(layer.feat_drop.p, 0.5)
        assert layer.lstm.input_size == 2
        assert layer.lstm.hidden_size == 2
        assert layer.fc_neigh.out_features == 3
        assert isinstance(layer.activation, torch.nn.LeakyReLU)

    def test_forward(self, dgl_methane):
        stack = SAGEConvStack.with_layers(
            n_input_features=4,
            hidden_feature_sizes=[2],
        )

        h = stack.forward(dgl_methane.homograph, dgl_methane.atom_features)
        assert h.detach().numpy().shape == (5, 2)

    def test_invalid_lengths(self):
        expected_err = (
            r"`layer_dropout` \(length 1\) must be a list of same length "
            r"as `hidden_feature_sizes` \(length 2\)."
        )
        with pytest.raises(ValueError, match=expected_err):
            SAGEConvStack.with_layers(
                n_input_features=1,
                hidden_feature_sizes=[2, 3],
                layer_dropout=[0.5],
            )

class TestDGLSageConv:
    def test_forward_values(self, dgl_methane):
        dgl = pytest.importorskip("dgl")
        layer = dgl.nn.pytorch.SAGEConv(
            in_feats=4,
            out_feats=3,
            aggregator_type="mean",
            feat_drop=0,
            activation=torch.nn.Sigmoid(),
            bias=False,
        )

        layer.fc_neigh.weight.data.fill_(1.0)
        layer.fc_self.weight.data.fill_(2.0)

        expected_features = np.array(
            [
                [0, 0, 0, 1],
                [1, 0, 0, 0],
                [1, 0, 0, 0],
                [1, 0, 0, 0],
                [1, 0, 0, 0],
            ]
        )
        assert_allclose(dgl_methane.atom_features.detach().numpy(), expected_features)

        results = layer.forward(dgl_methane.homograph, dgl_methane.atom_features)
        results = results.detach().numpy()
        assert results.shape == (5, 3)
        assert_array_almost_equal(results, 0.952574)

class TestSageConv:
    @pytest.fixture()
    def sageconv_layer(self):
        layer = SAGEConv(
            in_feats=4,
            out_feats=3,
            aggregator_type="mean",
            feat_drop=0,
            activation=torch.nn.Sigmoid(),
            bias=False,
        )

        layer.fc_neigh.weight.data.fill_(1.0)
        layer.fc_self.weight.data.fill_(2.0)
        return layer

    def test_forward_values_dgl(self, sageconv_layer, dgl_methane):
        expected_features = np.array(
            [
                [0, 0, 0, 1],
                [1, 0, 0, 0],
                [1, 0, 0, 0],
                [1, 0, 0, 0],
                [1, 0, 0, 0],
            ]
        )
        assert_allclose(dgl_methane.atom_features.detach().numpy(), expected_features)

        results = sageconv_layer.forward(
            dgl_methane.homograph, dgl_methane.atom_features
        )
        results = results.detach().numpy()
        assert results.shape == (5, 3)
        assert_array_almost_equal(results, 0.952574)

    def test_forward_values_dgl(self, sageconv_layer, nx_methane):
        expected_features = np.array(
            [
                [0, 0, 0, 1],
                [1, 0, 0, 0],
                [1, 0, 0, 0],
                [1, 0, 0, 0],
                [1, 0, 0, 0],
            ]
        )
        assert_allclose(nx_methane.atom_features.detach().numpy(), expected_features)

        results = sageconv_layer.forward(nx_methane.homograph, nx_methane.atom_features)
        results = results.detach().numpy()
        assert results.shape == (5, 3)
        assert_array_almost_equal(results, 0.952574)


# Docs on session basics

import numpy as np
import pandas as pd
import os
import sys
import pickle
from collections import OrderedDict

from flask import Flask, jsonify
#from flask_cors import CORS, cross_origin

#################################################
# read data from the csv files
#################################################

# print(pd.show_versions(), file=sys.stderr)
# Save reference to the table
weatherData = pd.read_csv("static/data/selected_weather_data_for_visual.csv")
aggData = pd.read_csv("static/data/agg_hist_311_data.csv")
censusData = pd.read_json("static/data/census_data.json")
weatherData['date_field_str'] = weatherData['date_field_str'].astype('str')
weatherData['yr'] = weatherData['yr'].astype('str')
censusData['zipcode'] = censusData['Zipcode'].astype('str')
aggData['zipcode'] = aggData['zipcode'].astype('str')
aggData['yr'] = aggData['yr'].astype('str')
aggData['date_field_str'] = aggData['date_field_str'].astype('str')
# aggData['date_field'] = aggData['date_field'].dt.date

# month = dict(one='January',
#                  two='February',
#                  three='March',
#                  four='April',
#                  five='May',
#                  six='June',
#                  seven='July',
#                  eight='August',
#                  nine='September',
#                  ten='October',
#                  eleven='November',
#                  twelve='December'
#                  )
# monthShort = dict(one='Jan',
#                  two='Feb',
#                  three='Mar',
#                  four='Apr',
#                  five='May',
#                  six='Jun',
#                  seven='Jul',
#                  eight='Aug',
#                  nine='Sep',
#                  ten='Oct',
#                  eleven='Nov',
#                  twelve='Dec'
#                  )

months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
monthsDict = {'Jan': 1,'Feb' : 2,'Mar': 3,'Apr': 4,'May': 5,'Jun': 6,'Jul': 7,'Aug': 8,'Sep': 9,'Oct': 10,'Nov': 11,'Dec': 12}

print(censusData.head(5), file=sys.stderr)
print(aggData.head(5), file=sys.stderr)

#################################################
# Flask Setup
#################################################
app = Flask(__name__)
#cors = CORS(app)
#app.config['CORS_HEADERS'] = 'Content-Type'

#################################################
# Loaf Model
#################################################

def load_model():
    global model
    with open('static/models/logistic_model.pkl', 'rb') as f:
        model = pickle.load(f)
        print("model loaded")

#################################################
# Flask Routes
#################################################

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        input_data = request.form.to_dict()
        data = process_input(input_data)
        value = model.predict(data)
        return render_template('index.html', result=value)

    return render_template('index.html')

@app.route("/api/v1.0/weather/date/<date_filter>")
def stateData(date_filter):
    """Return a list of all weather for the state"""

    # Query all passengers
    weather_sel = weatherData[weatherData['date_field_str'] == str(date_filter).strip()]
    print('data in filter:'+ str(date_filter), file=sys.stderr)
    # close the session to end the communication with the database
    # Convert list of tuples into normal list
#     all_names = list(np.ravel(results))
    all_data = []
    # for weather, index in weather_sel:
    for index, weather in weather_sel.iterrows():
        weather_dict = {}
        weather_dict["date"] = weather['date_field']
        weather_dict["tempMax"] = weather['tempMax']
        weather_dict["tempMin"] = weather['tempMin']
        weather_dict["tempAvg"] = weather['tempAvg']
        weather_dict["precipitation"] = weather['precipitation']

        all_data.append(weather_dict)

    return jsonify(all_data)

@app.route("/api/v1.0/weather/dateRange/<date_start>/<date_end>")
def all_states(date_start, date_end):
    """Return a list of all weather for the state"""

    # Query all passengers
    weather_sel = weatherData[weatherData['date_field_str'] >= date_start]
    weather_sel = weather_sel[weather_sel['date_field_str'] <= date_end]

    # close the session to end the communication with the database
    # Convert list of tuples into normal list
#     all_names = list(np.ravel(results))
    all_data = []
    # for weather, index in weather_sel:
    for index, weather in weather_sel.iterrows():
        weather_dict = {}
        weather_dict["date"] = weather['date_field']
        weather_dict["tempMax"] = weather['tempMax']
        weather_dict["tempMin"] = weather['tempMin']
        weather_dict["tempAvg"] = weather['tempAvg']
        weather_dict["precipitation"] = weather['precipitation']
        all_data.append(weather_dict)

    return jsonify(all_data)

@app.route("/api/v1.0/census/zipcode/<zipcode_filter>")
def zipCodeData(zipcode_filter):
    """Return a list of all weather for the state"""

    # Query all passengers
    census_sel = censusData[censusData['zipcode'] == str(zipcode_filter).strip()]
    print('data in filter:'+ str(zipcode_filter), file=sys.stderr)
    # close the session to end the communication with the database
    # Convert list of tuples into normal list
#     all_names = list(np.ravel(results))
    all_data = []
    # for weather, index in weather_sel:
    for index, census in census_sel.iterrows():
        census_dict = {}
        census_dict["zipcode"] = census['zipcode']
        census_dict["Population"] = census['Population']
        census_dict["MedianAge"] = census['Median Age']
        census_dict["HouseholdIncome"] = census['Household Income']
        census_dict["PerCapitaIncome"] = census['Per Capita Income']
        census_dict["PovertyRate"] = census['Poverty Rate']
        census_dict["TotalHouseholds"] = census['Total Households']
        census_dict["TotalOwnerOccupied"] = census['Total Owner Occupied']
        census_dict["PerOwnerOccupied"] = census['% Owner Occupied']

        all_data.append(census_dict)

    return jsonify(all_data)

@app.route("/api/v1.0/getAllZipCodes")
def getAllZipCodes():
    """Return a list of all weather for the state"""

    allZips = aggData.zipcode.unique()

    return jsonify(allZips.tolist())

@app.route("/api/v1.0/getAllTypes")
def getAllTypes():
    """Return a list of all weather for the state"""

    allServs = aggData.serv_type.unique()

    return jsonify(allServs.tolist())

@app.route("/api/v1.0/getAllDates")
def getAllDates():
    """Return a list of all weather for the state"""

    allDates = aggData.date_field_str.unique()

    return jsonify(allDates.tolist())

@app.route("/api/v1.0/getAllNeib")
def getAllNeib():
    """Return a list of all weather for the state"""

    allDates = aggData.neighborhood.unique()

    return jsonify(allDates.tolist())

@app.route("/api/v1.0/houston311/zipcode/date/type/<zipcode_filter>/<date_filter>/<type_filter>")
def houston311byAll(zipcode_filter, date_filter, type_filter):
    """Return a list of all weather for the state"""

    # Query all passengers
    agg_sel = aggData[aggData['zipcode'] == str(zipcode_filter).strip()]
    print('data in filter:'+ str(zipcode_filter), file=sys.stderr)
    agg_sel = agg_sel[agg_sel['date_field_str'] == str(date_filter).strip()]
    print('data in filter:'+ str(date_filter), file=sys.stderr)
    agg_sel = agg_sel[agg_sel['serv_type'] == str(type_filter).strip()]
    print('data in filter:'+ str(type_filter), file=sys.stderr)
    # close the session to end the communication with the database
    # Convert list of tuples into normal list
#     all_names = list(np.ravel(results))
    all_data = []
    # for weather, index in weather_sel:
    for index, census in agg_sel.iterrows():
        census_dict = {}
        census_dict["zipcode"] = census['zipcode']
        census_dict["Population"] = census['Population']
        census_dict["MedianAge"] = census['Median Age']
        census_dict["HouseholdIncome"] = census['Household Income']
        census_dict["PerCapitaIncome"] = census['Per Capita Income']
        census_dict["PovertyRate"] = census['Poverty Rate']
        census_dict["TotalHouseholds"] = census['Total Households']
        census_dict["TotalOwnerOccupied"] = census['Total Owner Occupied']
        census_dict["PerOwnerOccupied"] = census['% Owner Occupied']

        all_data.append(census_dict)

    return jsonify(all_data)

@app.route("/api/v1.0/houston311/top10Types")
def houston311Top10():
    """Return a list of all weather for the state"""
    aggTypes = aggData.groupby(['serv_type']).agg(
    # Get max of the duration column for each group
    count_issues=('serv_type', 'count'))
    aggTypes.reset_index(inplace=True)
    aggTypes.sort_values('count_issues',ascending=False,inplace=True)
    aggTypes = aggTypes.head(10)
    # close the session to end the communication with the database
    # Convert list of tuples into normal list
#     all_names = list(np.ravel(results))
    all_data = []
    # for weather, index in weather_sel:
    for index, row in aggTypes.iterrows():
        data_dict = {}
        data_dict["serv_type"] = row.serv_type
        data_dict["count_issues"] = row.count_issues

        all_data.append(data_dict)

    return jsonify(all_data)

@app.route("/api/v1.0/houston311/ByMonth")
def houston311top10ByMonth():
    """Return a list of all weather for the state"""
    aggTypes = aggData.groupby(['date_month']).agg(
    # Get max of the duration column for each group
    count_issues=('serv_type', 'count'))
    weathAgg = weatherData.groupby(['date_month']).agg(
    prec=('precipitation', 'sum'),
    avg_max=('tempMax', 'mean'))

    aggTypes.reset_index(inplace=True)
    weathAgg.reset_index(inplace=True)

    # close the session to end the communication with the database
    # Convert list of tuples into normal list
#     all_names = list(np.ravel(results))
    all_data = []
    # for weather, index in weather_sel:
    for index, row in aggTypes.iterrows():
        data_dict = {}
        data_dict["date_month"] = row.date_month
        data_dict["count_issues"] = row.count_issues
        for index1, row1 in weathAgg.iterrows():
            if (row1.date_month == row.date_month):
                data_dict["prec"] = row1.prec
                data_dict["avg_max"] = row1.avg_max

        all_data.append(data_dict)

    return jsonify(all_data)

@app.route("/api/v1.0/houston311/top10Types/zip/<zip_filter>/year/<year_filter>/neib/<neib_filter>/type/<type_filter>")
def houston311Top10byZip(zip_filter, year_filter, neib_filter,  type_filter):
    """Return a list of all weather for the state"""
    agg_sel = aggData
    if (zip_filter != 'ALL'):
        agg_sel = agg_sel[agg_sel['zipcode'] == str(zip_filter).strip()]
    if (year_filter != 'ALL'):
        agg_sel = agg_sel[agg_sel['yr'] == str(year_filter).strip()]
    if (neib_filter != 'ALL'):
        agg_sel = agg_sel[agg_sel['neighborhood'] == str(neib_filter).strip()]
    if (type_filter != 'ALL'):
        agg_sel = agg_sel[agg_sel['serv_type'] == str(type_filter).strip()]
    print('data in filter:'+ str(zip_filter), file=sys.stderr)
#    print('agg:'+ agg_sel.head(5), file=sys.stderr)
    aggTypes = agg_sel.groupby(['serv_type']).agg(
    # Get max of the duration column for each group
    count_issues=('serv_type', 'count'))
    aggTypes.reset_index(inplace=True)
    aggTypes.sort_values('count_issues',ascending=False,inplace=True)
    aggTypes = aggTypes.head(10)
    # close the session to end the communication with the database
    # Convert list of tuples into normal list
#     all_names = list(np.ravel(results))
    all_data = []
    # for weather, index in weather_sel:
    for index, row in aggTypes.iterrows():
        data_dict = {}
        data_dict["serv_type"] = row.serv_type
        data_dict["count_issues"] = row.count_issues

        all_data.append(data_dict)

    return jsonify(all_data)

@app.route("/api/v1.0/houston311/ByMonth/zip/<zip_filter>/year/<year_filter>/neib/<neib_filter>/type/<type_filter>")
def houston311top10ByMonthZip(zip_filter, year_filter, neib_filter, type_filter):
    """Return a list of all weather for the state"""
    agg_sel = aggData
    if (zip_filter != 'ALL'):
        agg_sel = agg_sel[agg_sel['zipcode'] == str(zip_filter).strip()]
    if (year_filter != 'ALL'):
        agg_sel = agg_sel[agg_sel['yr'] == str(year_filter).strip()]
    if (neib_filter != 'ALL'):
        agg_sel = agg_sel[agg_sel['neighborhood'] == str(neib_filter).strip()]
    if (type_filter != 'ALL'):
        agg_sel = agg_sel[agg_sel['serv_type'] == str(type_filter).strip()]

    print('data in filter:'+ str(zip_filter), file=sys.stderr)
    aggTypes = agg_sel.groupby(['date_month']).agg(
    count_issues=('serv_type', 'count'),
    time_taken=('avg_time', 'mean'),
    overdue=('avg_overdue', 'mean'))
    weath_sel = weatherData
    if (year_filter != 'ALL'):
        weath_sel = weatherData[weatherData['yr'] == str(year_filter).strip()]

    weathAgg = weath_sel.groupby(['date_month']).agg(
    prec=('precipitation', 'sum'),
    avg_max=('tempMax', 'mean'))

    aggTypes.reset_index(inplace=True)
    weathAgg.reset_index(inplace=True)
    # aggTypes.sort_values('count_issues',ascending=False,inplace=True)
    # aggTypes = aggTypes.head(10)
    # close the session to end the communication with the database
    # Convert list of tuples into normal list
#     all_names = list(np.ravel(results))
    all_data = []
    # for weather, index in weather_sel:
    for index, row in aggTypes.iterrows():
        data_dict = {}
        data_dict["sort_key"] = monthsDict[row.date_month]
        data_dict["date_month"] = row.date_month
        data_dict["count_issues"] = row.count_issues
        data_dict["time_taken"] = row.time_taken
        data_dict["overdue"] = row.overdue
        for index1, row1 in weathAgg.iterrows():
            if (row1.date_month == row.date_month):
                data_dict["prec"] = row1.prec
                data_dict["avg_max"] = row1.avg_max

        all_data.append(data_dict)
    #OrderedDict(sorted(all_data.items(),key =lambda x:months.index(x[0])))
    all_data = sorted(all_data, key = lambda i: (i['sort_key']))
    return jsonify(all_data)

@app.route("/api/v1.0/processModel/zip/<zip_filter>/temp/<temp_entered>/rain/<rain_entered>/type/<type_filter>")
def processModel(zip_filter, temp_entered, rain_entered, type_filter):
    """Return a list of all weather for the state"""

    census_sel = censusData[censusData['zipcode'] == str(zip_filter).strip()]
    print('data in filter:'+ str(zip_filter), file=sys.stderr)
    # close the session to end the communication with the database
    # Convert list of tuples into normal list
#     all_names = list(np.ravel(results))
    all_data = []
    data_dict = {}

    for index, census in census_sel.iterrows():
        population = census['Population']
        MedianAge = census['Median Age']
        HouseholdIncome = census['Household Income']
        PovertyRate = census['Poverty Rate']
        PerOwnerOccupied = census['% Owner Occupied']
        data_dict['MedianAge'] = census['Median Age']
        data_dict['HouseholdIncome'] = census['Household Income']
        data_dict["Population"] = census['Population']
        data_dict["PerCapitaIncome"] = census['Per Capita Income']
        data_dict["PovertyRate"] = census['Poverty Rate']
        data_dict["TotalHouseholds"] = census['Total Households']
        data_dict["TotalOwnerOccupied"] = census['Total Owner Occupied']
        data_dict["PerOwnerOccupied"] = census['% Owner Occupied']

    # weather_sel = weatherData[weatherData['date_field_str'] == str(date_selected).strip()]
    # print('data in filter:'+ str(date_selected), file=sys.stderr)
    # for index, weather in weather_sel.iterrows():
        # tempAvg = weather['tempAvg']
        # precipitation = weather['precipitation']

    contProb = 0
    if (type_filter == 'Container Problem'):
        contProb = 1
    drainProb = 0
    if (type_filter == 'Drainage'):
        drainProb = 1
    missedGarbProb = 0
    if (type_filter == 'Missed Garbage Pickup'):
        missedGarbProb = 1
    missedHeavyProb = 0
    if (type_filter == 'Missed Heavy Trash Pickup'):
        missedHeavyProb = 1
    missedRecProb = 0
    if (type_filter == 'Missed Recycling Pickup'):
        missedRecProb = 1
    nuisProb = 0
    if (type_filter == 'Nuisance On Property'):
        nuisProb = 1
    smwProb = 0
    if (type_filter == 'SWM Escalation'):
        smwProb = 1
    sewerProb = 0
    if (type_filter == 'Sewer Wasterwater'):
        sewerProb = 1
    stormProb = 0
    if (type_filter == 'Storm Debris Collection'):
        stormProb = 1
    streetCondProb = 0
    if (type_filter == 'Street Condition'):
        streetCondProb = 1
    streetHazardProb = 0
    if (type_filter == 'Street Hazard'):
        streetHazardProb = 1
    traficSignalProb = 0
    if (type_filter == 'Traffic Signal Maintenance'):
        traficSignalProb = 1
    trafficSignProb = 0
    if (type_filter == 'Traffic Sign'):
        trafficSignProb = 1
    waterLeakProb = 0
    if (type_filter == 'Water Leak'):
        waterLeakProb = 1
    waterServiceProb = 0
    if (type_filter == 'Water Service'):
        waterServiceProb = 1

# Population, Median Age, Household Income, Poverty Rate, % Owner Occupied, tempAvg, precipAvg, Container Problem, Drainage,
# Missed Garbage Pickup, Missed Heavy Trash Pickup, Missed Recycling Pickup, Nuisance On Property, SWM Escalation,
# Sewer Wasterwater, Storm Debris Collection, Street Condition,
# Street Hazard, Traffic Signal Maintenance, Traffic Sign, Water Leak, Water Service

    new_data = [[population, MedianAge, HouseholdIncome,
                PovertyRate, PerOwnerOccupied, float(temp_entered), float(rain_entered),
                contProb, drainProb, missedGarbProb, missedHeavyProb,
                missedRecProb, nuisProb, smwProb, sewerProb,
                stormProb, streetCondProb, streetHazardProb, traficSignalProb,
                trafficSignProb, waterLeakProb, waterServiceProb]]

    # new_data_scaled = X_scaler.transform(new_data)

    new_predict = model.predict(new_data)
    print(new_predict, file=sys.stderr)
    data_dict['Prediction'] = str(new_predict[0])
    all_data.append(data_dict)
    print(all_data, file=sys.stderr)

    #OrderedDict(sorted(all_data.items(),key =lambda x:months.index(x[0])))
    # all_data = sorted(all_data, key = lambda i: (i['sort_key']))
    return jsonify(all_data)

if __name__ == '__main__':
    load_model()
    app.run(debug=True)


from django.http import JsonResponse
from django.shortcuts import render,HttpResponseRedirect
from app.models import MainWheel,MainNav,MainMustBuy,MainShop,MainShow,FoodType,Goods,CartModel,OrderModel,OrderGoodsModel
from user.models import UserModel, UserTicketModel
from django.core.urlresolvers import reverse

from utils.functions import get_order_random_id

def home(request):
    '''
    首页视图函数
    '''
    if request.method == 'GET':
        mainwheels = MainWheel.objects.all()
        navs = MainNav.objects.all()
        mustbuys = MainMustBuy.objects.all()
        mainshops = MainShop.objects.all()
        mainshows = MainShow.objects.all()
        data = {
            'title':'首页',
            'mainwheel':mainwheels,
            'mustbuys':mustbuys,
            'navs':navs,
            'mainshops':mainshops,
            'mainshows':mainshows
        }
        return render(request,'home/home.html',data)

def mine(request):
    '''
    个人中心
    '''
    if request.method == 'GET':
        user = request.user
        orders = OrderModel.objects.filter(user=user)
        payed , wait_pay = 0,0
        for order in orders:
            if order.o_status == 0:
                wait_pay += 1
            if order.o_status == 1:
                payed +=1

        data = {
            'wait_pay':wait_pay,
            'payed':payed
        }
        return render(request,'mine/mine.html',data)

def market(request):
    '''
    闪购超市
    '''
    if request.method == 'GET':
        return HttpResponseRedirect(reverse('axf:params_market',args=('104749','0','0')))

def user_market(request,typeid,cid,sid):
    '''
    :param typeid: 分类id
    :param cid: 子分类id
    :param sid: 排序id
    '''
    if request.method == 'GET':
        ticket = request.COOKIES.get('ticket')
        user_ticket = UserTicketModel.objects.filter(ticket=ticket).first()
        if user_ticket:
            user = user_ticket.user
        else:
            user = ''
        # user = request.user
        foodtypes = FoodType.objects.all()
        if cid == '0':
            goods = Goods.objects.filter(categoryid=typeid)
        else:
            goods = Goods.objects.filter(categoryid=typeid,
                                         childcid=cid)
        foodtypes_current = foodtypes.filter(typeid=typeid).first()
        if foodtypes_current:
            childtypes = foodtypes_current.childtypenames
            childtypenames = childtypes.split('#')
            child_list = []
            for childtypename in childtypenames:
                child_type_info = childtypename.split(':')
                child_list.append(child_type_info)
        #排序
        if sid == '0':
            pass
        if sid == '1':
            goods = goods.order_by('productnum')
        if sid == '2':
            goods = goods.order_by('-price')
        if sid == '3':
            goods = goods.order_by('price')

        #返回购物车信息
        if user:
            user_cart = CartModel.objects.filter(user=user)
        else:
            user_cart = ''

        data = {
            'foodtypes':foodtypes,
            'goods':goods,
            'typeid':typeid,
            'child_list':child_list,
            'cid':cid,
            'user_cart':user_cart
        }
        return render(request,'market/market.html',data)

def add_cart(request):
    '''
    添加购物车
    '''
    if request.method == 'POST':
        user = request.user
        goods_id = request.POST.get('goods_id')
        #判断用户是否是系统自带的anonymouseuser还是登陆的用户
        data = {
            'code': 200,
            'msg': '请求成功'
        }
        if user.id:
            user_carts = CartModel.objects.filter(user=user,goods_id=goods_id).first()
            if user_carts:
                user_carts.c_num += 1
                user_carts.save()
                data['c_num'] = user_carts.c_num
            else:
                CartModel.objects.create(user=user,goods_id=goods_id)
                data['c_num'] = 1
            carts = CartModel.objects.filter(user=user,is_select=True)
            price = 0
            for cart in carts:
                price = cart.goods.price * cart.c_num + price
            data['price'] = int(price)
            return JsonResponse(data)
        data['msg'] = '当前用户没有登录'
        return JsonResponse(data)

def sub_cart(request):
    '''
    减少购物车用户下单商品的数量
    '''
    if request.method == 'POST':
        goods_id = request.POST.get('goods_id')
        user = request.user
        data = {
            'msg':'请求成功',
            'code':200
        }
        if user.id:
            #获取用户下单对应的商品信息
            user_carts = CartModel.objects.filter(user=user,goods_id=goods_id).first()
            #如果购物车中已经存在了商品信息
            if user_carts:
                if user_carts.c_num == 1 :
                    user_carts.delete()
                    data['c_num'] = 0
                else:
                    user_carts.c_num -= 1
                    data['c_num'] = user_carts.c_num
                    user_carts.save()
                price = 0
                carts = CartModel.objects.filter(user=user, is_select=True)
                for cart in carts:
                    price = cart.goods.price * cart.c_num + price
                data['price'] = int(price)
                return JsonResponse(data)
            data['c_num'] = 0
            return JsonResponse(data)
        data['msg'] = '该用户没有登录'
        data['code'] = '403'
        return JsonResponse(data)

def cart(request):
    if request.method == 'GET':
        user = request.user
        #查询购物车信息
        user_carts = CartModel.objects.filter(user=user)
        price = 0
        for carts in user_carts:
            if carts.is_select == True:
                price = carts.goods.price * carts.c_num + price
        data = {
            'user_carts':user_carts,
            'price':int(price)
        }
        return render(request,'cart/cart.html',data)

def change_select_status(request):
    if request.method == 'POST':
        user = request.user
        cart_id = request.POST.get('cart_id')
        cart = CartModel.objects.filter(id=cart_id).first()
        if cart.is_select:
            cart.is_select = False
        else:
            cart.is_select = True
        cart.save()
        g_carts = CartModel.objects.filter(user=user,is_select=True)
        price = 0
        for g_cart in g_carts:
            price = g_cart.goods.price * g_cart.c_num + price
        data = {
            'code':200,
            'msg':'请求成功',
            'is_select':cart.is_select,
            'price':int(price)
        }
        return JsonResponse(data)

def allcheck(request):
    if request.method == 'GET':
        user = request.user
        user_cart = CartModel.objects.filter(user=user)
        for cart in user_cart:
            cart.is_select = True
            cart.save()
        data = {
            'code':200,
            'msg':'请求成功',
        }
        return JsonResponse(data)

def generate_order(request):
    if request.method == 'GET':
        # 下单
        user = request.user
        # 创建订单
        o_num = get_order_random_id()
        order = OrderModel.objects.create(user=user,
                                  o_num=o_num)
        #选择勾选的商品进行下单
        user_carts = CartModel.objects.filter(user=user,is_select=True)
        for carts in user_carts:
            # 创建商品和订单之间的关系
            OrderGoodsModel.objects.create(goods=carts.goods,order=order,goods_num=carts.c_num)
        user_carts.delete()
        return render(request,'order/order_info.html',{'order':order})

def pay(request):
    if request.method == 'POST':
        order_id = request.POST.get('order_id')
        OrderModel.objects.filter(id = order_id).update(o_status = 1)
        return JsonResponse({'code':'200'})
        # user = request.user
        # order = OrderModel.objects.filter(user=user).first()
        # order.o_status = 1
        # order.save()
        # return render(request,'mine/mine.html')

def order_wait_pay(request):
    if request.method == 'GET':
        user = request.user
        orders = OrderModel.objects.filter(user=user,o_status=0)
        return render(request,'order/order_list_wait_pay.html',{'orders':orders})

def order_payed(request):
    if request.method == 'GET':
        user = request.user
        orders = OrderModel.objects.filter(user=user,o_status=1)
        return render(request,'order/order_list_payed.html',{'orders':orders})

def waitpay_topay(request):
    if request.method == 'GET':
        order_id = request.GET.get('order_id')
        order = OrderModel.objects.filter(id=order_id).first()
        return render(request,'order/order_info.html',{'order':order})


from samplesinus import SampleSinus
from vizualisation import Vizualisation
from closedform import ClosedForm
from matplotlib import pyplot as plt

class ModelSelection(object):
    """
    Fit different order polynomials to the training data.
    Evaluate model performance using a test data set.

    """

    FIG_W = 6  # Figure width
    FIG_H = 4  # Figure height

    BOX = [0.1, 0.1, 0.8, 0.8]       # Box axes 1
    YLIM = [0, 1]

    CLR_TRAIN  = '#003049'
    CLR_TEST = '#D62828'

    FRMT_TRAIN = '{0}//tranining_xte.dat'
    FRMT_TEST = '{0}//test_xte.dat'

    def __init__(self, Ntraining, Ntest, Mmax):
        """ Create test and training data set. Fit differt polynomials.

        :param Ntraining - int, number of data points in training dataset.
        :param Ntest - int, number of data points in test dataset.
        :param Mmax - int, maximum number for polynomial order.
        """

        # Create Training Data, Test Data and Function Definition
        FOLDER = 'model_N{0}'.format(Ntraining)
        self.train = self.FRMT_TRAIN.format(FOLDER)
        self.test = self.FRMT_TEST.format(FOLDER)

        sin = SampleSinus()
        sin.target(Ntraining, self.train)
        sin.target(Ntest, self.test, xdistr='random')
        SIN_FUNC = '{0}//sinus.func'.format(FOLDER)
        sin.function( SIN_FUNC )

        self.Ms = []
        self.RMSEs_train = []
        self.RMSEs_test = []

        self.Mmax= Mmax
        for m in range(self.Mmax+1):

            model = '{0}//poly_M{1}.func'.format(FOLDER, m)
            fig_name = '{0}//poly_M{1}.png'.format(FOLDER, m)
            weights_name = '{0}/weights_M{1}.dat'.format(FOLDER, m)

            cs = ClosedForm(m)
            cs.solve(self.train, weights_name)
            cs.test(self.test)
            cs.function(model)

            viz = Vizualisation()
            viz.target(self.train)
            viz.function( SIN_FUNC )
            viz.function(model, clr_option='model')
            viz.to_file( fig_name )

            self.Ms.append(m)
            self.RMSEs_train.append( cs.RMSE_train )
            self.RMSEs_test.append( cs.RMSE_test )

        file_out = '{0}//model_selection.png'.format( FOLDER )
        self._display_errors(file_out)

    def _display_errors(self, file_out):
        """ Display training and test errors. """

        self.FIG = plt.figure(figsize=(self.FIG_W, self.FIG_H))
        self.AX1 = self.FIG.add_axes(self.BOX)
        self.AX1.set_xlim([-1, self.Mmax+1])
        self.AX1.set_ylim(self.YLIM)

        self.AX1.plot(self.Ms, self.RMSEs_train,
                      marker='o',
                      linestyle='-',
                      linewidth=2,
                      color=self.CLR_TRAIN)
        self.AX1.plot(self.Ms, self.RMSEs_test,
                      marker='o',
                      linestyle='-',
                      linewidth=2,
                      color=self.CLR_TEST)

        self.FIG.savefig(file_out)