from tracer import logger
import json

class display:

    def __init__(self, printAll, jsonFormat):
        '''
        Initializes the attributes of the packetDict object

        Starts the packet dict with an empty dictionary

        The constructor for the packetDict class
        takes a packet object and a fieldName
        The field name is the first line of the header
        which details the HTTP method and URL

        Parameters:

        boolean printAll - If True, print all intercepted HTTP packets
        boolean jsonFormat - If True, print to stdout in newline-separated JSON objects
                             If False, print to stdout in pretty-printed format
                             Pretty-printed format uses newline-separated key-value pairs
                             whose keys are separated from their values by a colon and whitespace.

        Returns: packetDict
        '''
        self.printAll = printAll
        self.json = jsonFormat

        access_token         = 'access_token'
        access_token_expiry  = 'expires_in'
        refresh_token        = 'refresh_token'
        refresh_token_expiry = 'refresh_expires_in'
        token_type           = 'token_type'
        id_token             = 'id_token'
        not_before_policy    = 'not-before-policy'
        session_state        = 'session_state'

        packetSize           = 'packetSize'
        sourceIP             = 'sourceIP'
        destIP               = 'destIP'
        destPort             = 'destPort'
        sourcePort           = 'sourcePort'
        clientSecret         = 'clientSecret'
        grantType            = 'grantType'
        clientId             = 'clientId'
        httpHeader           = 'httpHeader'
        refreshToken         = 'refreshToken'
        authorizationCode    = 'authorizationCode'
        redirectUri          = 'redirectUri'
        scope                = 'scope'
        accessToken          = 'accessToken'
        accessTokenExpiry    = 'accessTokenExpiry'
        refreshToken         = 'refreshToken'
        refreshTokenExpiry   = 'refreshTokenExpiry'
        tokenType            = 'tokenType'
        idToken              = 'idToken'
        timestamp            = 'timestamp'

        # printList defines the default sequence to print
        # printList is a list of doubles
        # the first entry of a double is the pretty-printed name
        # and the second entry is the key to access the pDict for the value
        # keys that dont have entries are ignored in the pDict
        self.printList = [("Timestamp", timestamp),
                          ("Transmission Time", "http_time"),
                          ("Packet Size", packetSize),
                          ("Date", "http_date"),
                          ("Cookie", "http_cookie"),
                          ("User Agent", "http_user_agent"),
                          ("Connection", "http_connection"),
                          ("Request Full URI", "http_request_full_uri"),
                          ("Request Method", "http_request_method"),
                          ("Accept Language", "http_accept_language"),
                          ("Chat", "http_chat"),
                          ("Content Type", "content_type"),
                          ("Response Code", "response_code"),
                          ("Response Phrase", "response_phrase"),
                          ("Source IP", sourceIP),
                          ("Source Port", sourcePort),
                          ("Destination IP", destIP),
                          ("Destination Port", destPort),
                          ("Server", "server"),
                          ("Client Secret", clientSecret),
                          ("Client Id", clientId),
                          ("Grant Type", grantType),
                          ("Authorization Code", authorizationCode),
                          ("Redirect Uri", redirectUri),
                          ("Scope", scope),
                          ("Access Token", accessToken),
                          ("Access Token Expiry", accessTokenExpiry),
                          ("Refresh Token", refreshToken),
                          ("Refresh Token Expiry", refreshTokenExpiry),
                          ("Token Type", tokenType),
                          ("Id Token", idToken)]
        # ("File Data", "file_data")

        # a filter is a lambda
        # that takes parameter x which is a pDict
        # that when evaluated to True will cause
        # the display class to output the packet

        # filterToken is a predicate lambda that evaluates to True
        # only for packets that are responses containing tokens
        # or are requests for tokens
        self.filterToken = lambda x : self.index(x, refreshToken) or self.index(x, grantType) or self.index(x, accessToken) or self.index(x, idToken)

        # filterNone performs no filtering
        # and prints all (http) packets
        self.filterNone = lambda x : True

        # sets the filter to use
        # printAll switches the filter to print all (http) packets
        # the default is the token filter
        if self.printAll:
            self.filterCurrent = self.filterNone
        else:
            self.filterCurrent = self.filterToken

    def index(self, iDict, iKey):
        """
        Determines if iDict contains an entry for key iKey

        Parameters:

        dict iDict - a python dictionary
        str iKey - the key to index into iDict

        Returns True if dict iDict contains a value for key iKey
        Returns False on KeyError
        """
        try:
            iDict[iKey]
        except KeyError:
            return False
        return True

    def outPrint(self, pDict, fieldName, key):
        """
        Output function that prints the entry in pDict
        with key "key" with pretty-printed name "fieldName"

        Parameters:

        dict pDict
        str fieldName
        str key

        Returns:

        data
        """
        try:
            data = pDict[key]
        except KeyError:
            #logger.debug("KeyError on key {0}".format(key))
            return None
        print("{0:21}{1}".format(fieldName + ":", data))
        return data

    def prettyPrint(self, pDict):
        '''
        Pretty-prints the packet data to stdout

        Tightly coupled to the packetDict object

        Parameters:

        dict pDict - the dictionary representing the flattened packet data

        Returns: None
        '''
        logger.debug('Pretty-printing...')
        # printList is the sequence to print
        for item in self.printList:
            self.outPrint(pDict, item[0], item[1])
        print('')

    def jsonWrite(self, pDict):
        """
        Outputs JSON format to stdout

        Parameters:

        dict pDict - the dictionary representing the flattened packet data

        Returns: None
        """
        logger.debug('Outputting JSON')
        jsonString = json.JSONEncoder().encode(pDict)
        print(jsonString)

    def output(self, pDict):
        '''
        Main logging function

        Decides which format to print to stdout

        Parameters: None

        Returns: None
        '''
        logger.debug('Preparing to output...')
        packetTest = self.filterCurrent

        # if packetTest evaluates to True over the pDict
        # then output the packet
        if packetTest(pDict):
            if self.json:
                self.jsonWrite(pDict)
            else:
                self.prettyPrint(pDict)


# coding:utf-8
""""
Program: coucou
Description:
Author: XY - mailyanxin@gmail.com
Date: 2018-05-31 06:29:05
Last modified: 2018-05-31 06:29:05
Python release: 3.5.2
"""

from decimal import Decimal

from flask import current_app
from flask import jsonify
from flask import request
from flask.views import View
from flask.wrappers import Response
from flask_login import current_user

from coucou.extensions import db
from coucou.utils.helpers import decimal_2_float
from coucou.utils.helpers import set_float_prec

class MyPostView(View):
    '''
    输入参数预处理
    结果自动处理
    '''
    methods = ['POST', ]

    def __init__(self):
        data = request.get_json()
        self.params = self.preprocess_data(data)

        self.logger = current_app.logger
        self.debug = True

        if self.debug:
            self.logger.debug(self.params)

        self.token_name_id = self.params['token_name_id'] if 'token_name_id' in self.params else None
        if self.token_name_id == 'dacc_token':
            self.token_name_id = 'dacc'

        self.token_name = self.params['token_name'] if 'token_name' in self.params else None
        if self.token_name == 'dacc_token':
            self.token_name = 'dacc'
            self.params['token_name'] = 'dacc'

        self.phone_num = current_user.phone_num if not current_user.is_anonymous else None

        for k, v in self.params.items():
            setattr(self, k, v)

        super().__init__()

    def dispatch_request(self):

        self.ret_data = self.logic_view()

        # customize errcode
        if isinstance(self.ret_data, dict) and self.ret_data.get('errcode', 0) != 0:
            return jsonify(self.ret_data)
        # remark
        if isinstance(self.ret_data, dict) and\
            self.ret_data.get('errcode',1) == 0 and\
            self.ret_data.get('remark',None):
            return jsonify(self.ret_data)
        # None
        if self.ret_data is None:
            ret = dict(errcode=0, errmsg='ok')
            if self.debug:
                self.logger.debug(ret)
            return jsonify(ret)
        # 0 errcode
        if isinstance(self.ret_data, dict) or \
                isinstance(self.ret_data, list):
            ret = dict(errcode=0, errmsg='ok')
            self.after_process_data()
            ret['data'] = self.ret_data
            if self.debug:
                self.logger.debug(ret)
            return jsonify(ret)
        # login required ..
        if isinstance(self.ret_data, Response):
            return self.ret_data

        else:
            self.logger.info(self.ret_data)
            self.logger.info(type(self.ret_data))
            return self.ret_data

    def preprocess_data(self, data):

        column_type_info = dict()
        for table_name, table_obj in db.metadata.tables.items():
            colums = table_obj.columns.items()
            for i in colums:
                column_type_info[i[0]] = i[1].type

        new_data = dict()
        for k, v in data.items():
            k_type = column_type_info.get(k, None)
            if k_type:
                if isinstance(k_type, db.String):
                    v = str(v)
                if isinstance(k_type, db.Integer):
                    v = int(v)
                if isinstance(k_type, db.Float):
                    v = float(v)
                if isinstance(k_type, db.DECIMAL):
                    if isinstance(v, float):
                        v = Decimal(str(v))
                    else:
                        v = Decimal(v)

            new_data[k] = v

        return new_data

    def after_process_data(self):

        if isinstance(self.ret_data, dict):
            new_data = dict()
            for k, v in self.ret_data.items():
                if isinstance(v, Decimal):
                    new_data[k] = decimal_2_float(v)
                if isinstance(v, float):
                    if 'star' in k:
                        new_data[k] = set_float_prec(v)
                    else:
                        new_data[k] = v
                else:
                    new_data[k] = v

            self.ret_data = new_data
        else:
            pass


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /tmp/pip-build-ed191__6/requests-toolbelt/requests_toolbelt/auth/http_proxy_digest.py
# Compiled at: 2020-01-10 16:25:32
# Size of source mod 2**32: 3705 bytes
"""The module containing HTTPProxyDigestAuth."""
import re
from requests import cookies, utils
from . import _digest_auth_compat as auth

class HTTPProxyDigestAuth(auth.HTTPDigestAuth):
    __doc__ = 'HTTP digest authentication between proxy\n\n    :param stale_rejects: The number of rejects indicate that:\n        the client may wish to simply retry the request\n        with a new encrypted response, without reprompting the user for a\n        new username and password. i.e., retry build_digest_header\n    :type stale_rejects: int\n    '
    _pat = re.compile('digest ', flags=(re.IGNORECASE))

    def __init__(self, *args, **kwargs):
        (super(HTTPProxyDigestAuth, self).__init__)(*args, **kwargs)
        self.stale_rejects = 0
        self.init_per_thread_state()

    @property
    def stale_rejects(self):
        thread_local = getattr(self, '_thread_local', None)
        if thread_local is None:
            return self._stale_rejects
        else:
            return thread_local.stale_rejects

    @stale_rejects.setter
    def stale_rejects(self, value):
        thread_local = getattr(self, '_thread_local', None)
        if thread_local is None:
            self._stale_rejects = value
        else:
            thread_local.stale_rejects = value

    def init_per_thread_state(self):
        try:
            super(HTTPProxyDigestAuth, self).init_per_thread_state()
        except AttributeError:
            pass

    def handle_407(self, r, **kwargs):
        """Handle HTTP 407 only once, otherwise give up

        :param r: current response
        :returns: responses, along with the new response
        """
        if r.status_code == 407 and self.stale_rejects < 2:
            s_auth = r.headers.get('proxy-authenticate')
            if s_auth is None:
                raise IOError('proxy server violated RFC 7235:407 response MUST contain header proxy-authenticate')
            else:
                if not self._pat.match(s_auth):
                    return r
            self.chal = utils.parse_dict_header(self._pat.sub('', s_auth, count=1))
            if 'Proxy-Authorization' in r.request.headers:
                if 'stale' in self.chal:
                    if self.chal['stale'].lower() == 'true':
                        self.stale_rejects += 1
                    elif self.chal['stale'].lower() == 'false':
                        raise IOError('User or password is invalid')
            r.content
            r.close()
            prep = r.request.copy()
            cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
            prep.prepare_cookies(prep._cookies)
            prep.headers['Proxy-Authorization'] = self.build_digest_header(prep.method, prep.url)
            _r = (r.connection.send)(prep, **kwargs)
            _r.history.append(r)
            _r.request = prep
            return _r
        else:
            return r

    def __call__(self, r):
        self.init_per_thread_state()
        if self.last_nonce:
            r.headers['Proxy-Authorization'] = self.build_digest_header(r.method, r.url)
        r.register_hook('response', self.handle_407)
        return r


import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.preprocessing import StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.svm import SVC
from sklearn.linear_model import SGDRegressor
from sklearn.metrics import roc_auc_score, r2_score

# import the data file
df = pd.read_csv('train_features.csv')
dy = pd.read_csv('train_labels.csv')
dt = pd.read_csv('test_features.csv')

#---Compare the number of missing values in each column---
#missing = df.isnull().sum(0).reset_index()
#missing.columns = ['column', 'count']
#missing = missing.sort_values(by = 'count', ascending = False).loc[missing['count'] > 0]
#missing['percentage'] = missing['count'] / float(df.shape[0]) * 100
#ind = np.arange(missing.shape[0])
#width = 1
#fig, ax = plt.subplots(figsize=(8,10))
#rects = ax.barh(ind, missing.percentage.values, color='b')
#ax.set_yticks(ind)
#ax.set_yticklabels(missing.column.values, rotation='horizontal')
#ax.set_xlabel("Precentage of missing values %", fontsize = 10)
#ax.set_title("Number of missing values in each column", fontsize = 12)
#plt.show()
#--------------------------------------------------------

#-----------------Correlation diagram--------------------
#colormap = plt.cm.RdBu
#plt.figure(figsize=(24,22))
#plt.title('Pearson Correlation of Features', y=1.05, size=15)
#sns.heatmap(X_trains.astype(float).corr(),linewidths=0.1, vmax=1.0, \
#            square=True, cmap=colormap, linecolor='white', annot=True)
#--------------------------------------------------------

# assign X and y from the train dataset
all_pids = df.groupby('pid')
ap = list(all_pids.groups.keys())
train_pids, val_pids = train_test_split(ap, test_size = 0.2, \
                                        random_state = 42)

vital = ['Heartrate', 'SpO2', 'ABPs', 'ABPm', 'RRate','ABPd']
test = ['Temp', 'EtCO2', 'PTT', 'BUN', 'Lactate', 'Hgb', 'HCO3', 'BaseExcess', \
         'Fibrinogen', 'Phosphate', 'WBC', 'Creatinine', 'PaCO2', 'AST', \
         'FiO2', 'Platelets', 'SaO2', 'Glucose', 'Magnesium', 'Potassium', \
         'Calcium', 'Alkalinephos', 'Bilirubin_direct', 'Chloride', \
         'Hct', 'Bilirubin_total', 'TroponinI', 'pH']
tl = ['LABEL_BaseExcess', 'LABEL_Fibrinogen', 'LABEL_AST', 'LABEL_Alkalinephos', \
      'LABEL_Bilirubin_total', 'LABEL_Lactate', 'LABEL_TroponinI', 'LABEL_SaO2', \
      'LABEL_Bilirubin_direct', 'LABEL_EtCO2', 'LABEL_Sepsis']
vl = ['LABEL_RRate', 'LABEL_ABPm', 'LABEL_SpO2', 'LABEL_Heartrate']

#X_train = pd.DataFrame()
#X_val = pd.DataFrame()
#Y_train = pd.DataFrame()
#Y_val = pd.DataFrame()
#itt = IterativeImputer(max_iter=20, tol=0.01, random_state = 42)
#preprocessor = ColumnTransformer(transformers = [ \
#                                        ('vital', itt, vital)])
#sc = StandardScaler()
#
##Imputing missing values
#X_i = df.copy()
#X_i.loc[:,test] = X_i.loc[:,test].notnull().astype('float')
#imp = preprocessor.fit(X_i)
#X_i.loc[:,vital] = imp.transform(X_i)
#X_i = X_i.drop(['Bilirubin_total', 'PaCO2', 'BaseExcess', 'Alkalinephos', \
#             'HCO3', 'PTT', 'Phosphate', 'Magnesium', 'Creatinine', \
#             'Calcium', 'Platelets', 'WBC'], axis = 1)
#
#for pid in train_pids:
#    X_pid = X_i.groupby('pid').get_group(pid)
#    X_pid = X_pid.drop(['pid', 'Time'], axis=1)
#    xp = X_pid.iloc[11].values
#    xi = X_pid.iloc[:,:].drop(['Age'], axis=1).values.flatten()
#    xj = pd.DataFrame([[*xi, *xp]])
#    X_train = X_train.append(xj)
#X_trains = pd.DataFrame(sc.fit_transform(X_train))
#
#for pid in val_pids:
#    X_pid = X_i.groupby('pid').get_group(pid)
#    X_pid = X_pid.drop(['pid', 'Time'], axis=1)
#    xp = X_pid.iloc[11].values
#    xi = X_pid.iloc[:,:].drop(['Age'], axis=1).values.flatten()
#    xj = pd.DataFrame([[*xi, *xp]])
#    X_val = X_val.append(xj)
#X_vals = pd.DataFrame(sc.fit_transform(X_val))
#
#
##Split y values into train and validation set
#ypids = dy.groupby('pid')
#for pid in train_pids:
#    Y_pid = ypids.get_group(pid)
#    Y_pid = Y_pid.drop(['pid'], axis=1)
#    Y_train = Y_train.append(Y_pid)
#
#for pid in val_pids:
#    Y_pid = ypids.get_group(pid)
#    Y_pid = Y_pid.drop(['pid'], axis=1)
#    Y_val = Y_val.append(Y_pid)
#
##Test set
#X_t = dt.copy()
#X_t.loc[:,test] = X_t.loc[:,test].notnull().astype('float')
#X_t.loc[:,vital] = imp.transform(X_t)
#X_t = X_t.drop(['Bilirubin_total', 'PaCO2', 'BaseExcess', 'Alkalinephos', \
#             'HCO3', 'PTT', 'Phosphate', 'Magnesium', 'Creatinine', \
#             'Calcium', 'Platelets', 'WBC'], axis = 1)
#
#test_pid = list(dt.groupby('pid').groups.keys())
#X_test = pd.DataFrame()
#for pid in test_pid:
#    X_pid = X_t.groupby('pid').get_group(pid)
#    X_pid = X_pid.drop(['pid', 'Time'], axis=1)
#    xp = X_pid.iloc[11].values
#    xi = X_pid.iloc[:,:].drop(['Age'], axis=1).values.flatten()
#    xj = pd.DataFrame([[*xi, *xp]])
#    X_test = X_test.append(xj)
#X_tests = pd.DataFrame(sc.fit_transform(X_test))

svc = SVC(kernel='rbf', C=0.1, class_weight='balanced', gamma='scale', \
          probability=True, random_state=0)
svr = SGDRegressor(penalty='elasticnet', alpha=0.05, l1_ratio=0.1)
Y_predd = pd.DataFrame()
Y_predd = Y_predd.append([pd.DataFrame(test_pid, columns=['pid'])])

#for i in ['LABEL_Sepsis']:
#    svc.fit(X_trains, Y_train[i])
#    y_pred = svc.predict(X_vals)
#    print('roc_auc_score:', roc_auc_score(Y_val[i], y_pred))

for i in tl:
    svc.fit(X_trains, Y_train[i])
    y_pred = svc.predict_proba(X_tests)
    Y_predd[i] = pd.DataFrame(y_pred)[1]

#for i in vl:
#    svr.fit(X_trains, Y_train[i])
#    y_pred = svr.predict(X_vals)
#    print('r2 score', r2_score(Y_val[i], y_pred))

for i in vl:
    svr.fit(X_trains, Y_train[i])
    y_pred = svr.predict(X_tests)
    Y_predd[i] = pd.DataFrame(y_pred)

Y_predd.to_csv('prediction.zip', index=False, float_format='%.3f', compression='zip')


"""empty message

Revision ID: a0fbd4d0f7c3
Revises:
Create Date: 2020-11-20 15:39:08.444169

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'a0fbd4d0f7c3'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('player',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=80), nullable=False),
    sa.Column('club', sa.String(length=120), nullable=False),
    sa.Column('age', sa.Integer(), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('player')
    # ### end Alembic commands ###


from serial import Serial
from config import (
    SINGLE_PRESS_DURATION,
    MIN,
    CENTER,
    MAX,
    Y,
    B,
    A,
    X,
    L,
    R,
    ZL,
    ZR,
    MINUS,
    PLUS,
    LCLICK,
    RCLICK,
    HOME,
    HAT_CENTER,
)
import time

def SplitHex(num):
    return [(num >> 8), (num & 0xFF)]

class Command:
    def __init__(
        self, lx=CENTER, ly=CENTER, rx=CENTER, ry=CENTER, hat=HAT_CENTER, button=0
    ):
        self.lx = lx
        self.ly = ly
        self.rx = rx
        self.ry = ry
        self.hat = hat
        self.button = button

    def __eq__(self, other):
        if isinstance(other, Command):
            return (
                self.lx == other.lx
                and self.ly == other.ly
                and self.rx == other.rx
                and self.ry == other.ry
                and self.hat == other.hat
                and self.button == other.button
            )
        return False

    def as_byte(self):
        button = SplitHex(self.button)
        return bytearray(
            [self.lx, self.ly, self.rx, self.ry, self.hat, button[0], button[1]]
        )

class Serial(Serial):
    def __init__(self, *args, **kwargs):
        self.p_command = None
        self.start_timer = 0
        self.stop_timer = 0
        self.csv_writer = None
        return super().__init__(*args, **kwargs)

    def attach_csv(self, csv_writer):
        self.csv_writer = csv_writer

    def my_write(self, command: Command):
        if self.p_command == None:
            self.p_command = command
            self.start_timer = time.time_ns()

        if self.p_command == command:
            self.counter = time.time()
            self.stop_timer = time.time_ns()
        else:
            self.csv_writer.writerow(
                [
                    self.p_command.lx,
                    self.p_command.ly,
                    self.p_command.rx,
                    self.p_command.ry,
                    self.p_command.hat,
                    self.p_command.button,
                    (self.stop_timer - self.start_timer),
                ]
            )
            self.start_timer = time.time_ns()
            self.stop_timer = self.start_timer
            self.p_command = command

        self.write(command.as_byte())


from __future__ import absolute_import

from ..vtask import VTask

from thrift.server.TNonblockingServer import TNonblockingServer
from thrift.transport.TSocket import TServerSocket

import time

class ThriftProcessorTask(VTask):
    LOOPLESS = True
    PROCESSOR = None

    def __init__(self, service):
        super(ThriftProcessorTask, self).__init__(service)
        assert self.PROCESSOR is not None
        self.processor = self.PROCESSOR(self.service)

class NBServerTask(VTask):
    DEFAULT_HOST = '0.0.0.0'
    DEFAULT_PORT = 0
    OPT_PREFIX = 'thrift'

    bound_host = bound_port = None

    def getProcessor(self):
        """Automatically find the ThriftProcessorTask subclass"""
        found = None
        for task in self.service.tasks:
            if isinstance(task, ThriftProcessorTask):
                assert found is None, "Multiple processor tasks! (%s, %s)" % \
                    (found.name, task.name)
                found = task
        assert found is not None, "No ThriftProcessorTask's found!"
        return found.processor

    def initTask(self):
        super(NBServerTask, self).initTask()

        self._stopped = False
        self.socket = TServerSocket(
            self.getTaskOption('host'), self.getTaskOption('port'))
        self.server = TNonblockingServer(
            self.getProcessor(), self.socket,
            threads=self.getTaskOption('threads'))
        self.server.prepare()
        self.bound_host, self.bound_port = \
            self.server.socket.handle.getsockname()
        self.logger.info("%s Server Started on %s:%s",
                         self.name, self.bound_host, self.bound_port)

    def stop(self):
        self.server.stop()
        self.server.close()
        self._stopped = True

    def _runloop(self):
        while not self.server._stop:
            self.server.serve()
        while not self._stopped:
            time.sleep(0.1)

    @classmethod
    def _addArguments(cls, ap):
        super(NBServerTask, cls)._addArguments(ap)
        ap.add_argument(cls._loptName('host'), default=cls.DEFAULT_HOST,
                        metavar='HOST',
                        help='Address to bind server to [%(default)s]')
        ap.add_argument(cls._loptName('port'), type=int, metavar='PORT',
                        default=cls.DEFAULT_PORT,
                        help='Port to run server on [%(default)s]')
        ap.add_argument(cls._loptName('threads'), type=int, default=10,
                        metavar='N',
                        help='Server Worker Threads [%(default)s]')


import mock
import os
import unittest

from ngi_pipeline.engines.sarek.exceptions import ParserException
from ngi_pipeline.engines.sarek.models.sample import SarekAnalysisSample
from ngi_pipeline.engines.sarek.models.workflow import SarekWorkflowStep, SarekMainStep
from ngi_pipeline.tests.engines.sarek.models.test_sarek import TestSarekGermlineAnalysis

class TestSarekWorkflowStep(unittest.TestCase):

    CONFIG = TestSarekGermlineAnalysis.CONFIG

    def setUp(self):
        self.sarek_args = self.CONFIG["sarek"].copy()
        self.sarek_workflow_step = SarekWorkflowStep(
            self.sarek_args["command"],
            **{k: v for k, v in self.sarek_args.items() if k != "command"})

    def test__append_argument(self):
        base_string = "this-is-the-base-string"

        # test a non-existing attribute
        attribute = "this-attribute-does-not-exist"
        self.assertEqual(base_string, self.sarek_workflow_step._append_argument(base_string, attribute))

        # test a None attribute
        attribute = "none_attribute"
        setattr(self.sarek_workflow_step, attribute, None)
        self.assertEqual(base_string, self.sarek_workflow_step._append_argument(base_string, attribute))

        # test a list attribute
        attribute = "list_attribute"
        value = ["this", "is", "a", "list"]
        self.sarek_workflow_step.parameters[attribute] = value
        expected_result = "{0} --{1} ${{{1}}}".format(base_string, attribute)
        self.assertEqual(expected_result, self.sarek_workflow_step._append_argument(base_string, attribute))

        # test a string attribute
        attribute = "string_attribute"
        value = "this-is-a-string"
        self.sarek_workflow_step.parameters[attribute] = value
        expected_result = "{0} --{1} ${{{1}}}".format(base_string, attribute)
        self.assertEqual(expected_result, self.sarek_workflow_step._append_argument(base_string, attribute))

        # test a custom hyphen string
        hyphen = "xyz"
        expected_result = "{0} {2}{1} ${{{1}}}".format(base_string, attribute, hyphen)
        self.assertEqual(
            expected_result, self.sarek_workflow_step._append_argument(
                base_string, attribute, hyphen=hyphen))

    def test_sarek_step(self):
        with self.assertRaises(NotImplementedError):
            self.sarek_workflow_step.sarek_step()

    def test_command_line(self):
        sarek_workflow_step = SarekWorkflowStep(self.sarek_args["command"])
        self.assertEqual(self.sarek_args["command"], sarek_workflow_step.command_line())

    def test_command_line_args(self):
        valid_tools = self.sarek_args["tools"]
        observed_command_line = self.sarek_workflow_step.command_line()
        self.assertIn("--tools", observed_command_line)
        self.assertIn(",".join(valid_tools), observed_command_line)
        for key in [x for x in list(self.sarek_args.keys()) if x not in ["tools", "command"]]:
            self.assertIn("-{} {}".format(key, self.sarek_args[key]), observed_command_line)

class TestSarekMainStep(unittest.TestCase):

    def test_report_files(self):
        analysis_sample = mock.Mock(spec=SarekAnalysisSample)
        analysis_sample.sampleid = "this-is-a-sample-id"
        analysis_sample.sample_analysis_path.return_value = "this-is-a-path"
        analysis_sample.sample_analysis_results_dir.return_value = "this-is-the-results-path"
        with mock.patch("os.listdir") as list_mock:
            file_list = ["file1", "file2.extension", "file3.metrics", "file4.metrics"]
            for listed_files in [file_list[0:2], file_list]:
                list_mock.return_value = listed_files
                with self.assertRaises(ParserException):
                    SarekMainStep.report_files(analysis_sample)
            list_mock.return_value = file_list[0:3]
            self.assertEqual(
                file_list[2], os.path.basename(SarekMainStep.report_files(analysis_sample)[1][1]))


from time import time
from humanize import naturaldelta
from ensmallen_graph import EnsmallenGraph  # pylint: disable=no-name-in-module
import compress_json
import json
import numpy as np

start = time()
graph = EnsmallenGraph(
    edge_path="../graph/cooccurrence/edges.tsv",
    sources_column="subject",
    destinations_column="object",
    directed=False,
    validate_input_data=True
)
completed_graph = time() - start
start_walk = time()

walks = graph.walk(
    iterations=1,
    length=80,
    return_weight=1,
    explore_weight=1,
    change_node_type_weight=1,
    change_edge_type_weight=1
)
delta = time() - start
total_walk_time = time() - start_walk

mean_walks_length = np.mean([
    len(walk) for walk in walks
])

median_walks_length = np.median([
    len(walk) for walk in walks
])

degrees = [
    graph.degree(node)
    for node in range(graph.get_nodes_number())
]

response = {
    "directory": "cooccurrence",
    "total_required_time": delta,
    "building_graph_required_time": completed_graph,
    "random_walk_time": total_walk_time,
    "mean_walks_length": mean_walks_length,
    "median_walks_length": median_walks_length,
    "traps_rate": graph.traps_rate(),
    "mean_outbound_edges": np.mean(degrees),
    "median_outbound_edges": np.median(degrees),
    "nodes": graph.get_nodes_number(),
    "edges": graph.get_edges_number()
}

print(json.dumps(response, indent=4))

compress_json.dump(response, "time_required.json", json_kwargs={"indent": 4})


# 我采用requests库
import requests
import time

# 用来获取 时间戳
def gettime():
    return int(round(time.time() * 1000))

if __name__ == '__main__':
    # 用来自定义头部的
    headers = {}
    # 用来传递参数的
    keyvalue = {}
    # 目标网址(问号前面的东西)
    url = 'http://data.stats.gov.cn/easyquery.htm'

    # 头部的填充
    headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14) ' \
                            'AppleWebKit/605.1.15 (KHTML, like Gecko) ' \
                            'Version/12.0 Safari/605.1.15'

    # 下面是参数的填充，参考图10
    keyvalue['m'] = 'QueryData'
    keyvalue['dbcode'] = 'hgnd'
    keyvalue['rowcode'] = 'zb'
    keyvalue['colcode'] = 'sj'
    keyvalue['wds'] = '[]'
    keyvalue['dfwds'] = '[{"wdcode":"zb","valuecode":"A0301"}]'
    keyvalue['k1'] = str(gettime())

    # 发出请求，使用get方法，这里使用我们自定义的头部和参数
    # r = requests.get(url, headers=headers, params=keyvalue)
    # 建立一个Session
    s = requests.session()
    # 在Session基础上进行一次请求
    r = s.get(url, params=keyvalue, headers=headers)
    # 打印返回过来的状态码
    print (r.status_code)
    # 修改dfwds字段内容
    keyvalue['dfwds'] = '[{"wdcode":"sj","valuecode":"2000"}]'
    # 再次进行请求
    r = s.get(url, params=keyvalue, headers=headers)
    # 此时我们就能获取到我们搜索到的数据了
    print (r.text)


'''
@author:  etekinalp
@date:    Aug 26, 2014
@mail:    e.tekinalp@icloud.com
@brief:   This module setups a plugin
'''

from maya import cmds, mel
from goe_plugins import plugin_master
reload(plugin_master)

class Plug(plugin_master.PluginSetup):
    """
    Subclass PluginSetup and setup the plugin test environment.
    """

    def __init__(self, plugin, name):
        """
        @param plugin(string): Plugin name without .so or .py suffix
        @param name(string): Name of plugin call
        """

        super(Plug, self).__init__(plugin, 'so', True, True)

        # args
        self.plugin = plugin
        self.name = name

        # methods
        self._setup_plugin()
    # END __init__()

    def _setup_plugin(self):
        cmds.createNode(self.name)
    # END _setup_plugin()
# END Plug()

Plug("curveramp", "curveRamp")


import sys
sys.path.append('..')

from models.base_class import Model
from activations import relu
from layers import conv2d, batch_normalization, fully_connected, max_pooling, avg_pooling, global_avg_pooling
import tensorflow.compat.v1 as tf

class SKResNeXt(Model):
    STRUCTURES = {
        29: {
            'block_nums': [3, 3, 3],
            'cardinality': 16
        }
    }

    FIRST_CONV_OUT_CHANNEL =  64
    BOTTLENECK_OUT_CHANNEL = 4
    BOTTLENECK_CHANNEL_EXPANSION = 4
    CONV_KERNEL = 3
    REDUCTION = 16
    L = 32
    M = [3, 1]

    def __init__(self, depth, class_num, is_small):
        self.depth = depth
        self.class_num = class_num
        self.is_small = is_small

        self.structure = self.STRUCTURES[self.depth]['block_nums']
        self.cardinality = self.STRUCTURES[self.depth]['cardinality']

    def sk_module(self, x, out_channel, stride, name):
        with tf.variable_scope(name):
            u_list = list()
            with tf.variable_scope('split'):
                # 对同一输入使用不同大小的卷积核完成卷积
                for idx, k in enumerate(self.M):
                    u_list.append(
                        conv2d(x,
                               out_channel,
                               kernel=k,
                               stride=stride,
                               padding='SAME',
                               name='conv_{}'.format(idx))
                    )
                # 将所有不同尺度的特征连接成一个张量
                # u_list形状为[num_fea, batch_size, H', W', C_out]
                u_list = tf.stack(u_list, axis=0)

            with tf.variable_scope('fuse'):
                # 把所有不同尺度的特征按位加得到融合特征
                # u的形状为[batch_size, H', W', C_out]
                u = tf.reduce_sum(u_list, axis=0, name='sum')

                # 将融合的特征进行压缩
                s = global_avg_pooling(u, name='global_avg_pool')

                # 计算全连接层输出的特征数
                fc_out = max(out_channel // self.REDUCTION, self.L)
                z = fully_connected(s,
                                    fc_out,
                                    name='fully_connected')
                z = relu(z, name='relu')

            attention_list = list()
            with tf.variable_scope('select'):
                # 对每一个不同尺度的特征计算相应的权重
                for idx in range(len(self.M)):
                    attention_list.append(
                        fully_connected(z,
                                        out_channel,
                                        name='fully_connected_{}'.format(idx))
                    )
                # 将所有不同尺度特征的权重连接成一个张量
                # attention_list形状为[num_fea, batch_size, C_out]
                attention_list = tf.stack(attention_list, axis=0)

                # 不同尺度特征的权重之间使用Softmax互相抑制
                attention_list = tf.math.softmax(attention_list,
                                                 axis=0,
                                                 name='softmax')

                # 为了方便按位乘，将其形状变为[num_fea, batch_size, 1, 1, C_out]
                attention_list = tf.expand_dims(attention_list, axis=2)
                attention_list = tf.expand_dims(attention_list, axis=2)

                # 使用特征与其对应的权重按位乘
                # output形状为[num_fea, batch_size, H', W', C_out]
                output = u_list * attention_list

                # 将按位乘的结果按位加进行融合
                # output形状为[batch_size, H', W', C_out]
                output = tf.reduce_sum(output, axis=0, name='merge')

            return output

    def bottleneck(self, x, out_channel, stride, name):
        def transform(x, out_channel, stride):
            x_list = tf.split(x,
                              num_or_size_splits=self.cardinality,
                              axis=-1, name='split')
            out_channel = out_channel // self.cardinality

            # 将ResNeXt中的Bottleneck的分组卷积操作全部替换为SK模块
            for idx, x in enumerate(x_list):
                with tf.variable_scope('group_conv_{}'.format(idx)):
                    x = self.sk_module(x,
                                       out_channel,
                                       stride=stride,
                                       name='sk_module_{}'.format(idx))
                    x = batch_normalization(x,
                                            name='bn',
                                            is_training=self.is_training)
                    x = relu(x, name='relu')
                    out_x = x if idx == 0 else tf.concat([out_x, x], axis=-1)

            return out_x

        with tf.variable_scope(name):
            # 第1个1*1卷积
            output = conv2d(x,
                            out_channel,
                            kernel=1,
                            stride=1, name='conv1')
            output = batch_normalization(output,
                                         name='bn1',
                                         is_training=self.is_training)
            output = relu(output, name='relu')

            # SK模块
            with tf.variable_scope('transform'):
                output = transform(output,
                                   out_channel=out_channel,
                                   stride=stride)

            # 第3个1*1卷积
            output = conv2d(output,
                            out_channel * self.BOTTLENECK_CHANNEL_EXPANSION,
                            kernel=1,
                            stride=1,
                            padding='SAME', name='conv2')
            output = batch_normalization(output,
                                         name='bn2',
                                         is_training=self.is_training)

            c_in = x.get_shape().as_list()[-1]
            c_out = out_channel * self.BOTTLENECK_CHANNEL_EXPANSION

            with tf.variable_scope('shortcut'):
                if stride != 1 or c_in != c_out:
                    shortcut = conv2d(x,
                                      out_channel * self.BOTTLENECK_CHANNEL_EXPANSION,
                                      kernel=1,
                                      stride=stride,
                                      padding='SAME', name='conv')
                    shortcut = batch_normalization(shortcut,
                                                   name='bn',
                                                   is_training=self.is_training)
                else:
                    shortcut = x

            output = output + shortcut
            output = relu(output, name='relu')

            return output

    def build(self, x, is_training):
        self.is_training = is_training

        with tf.variable_scope('skresnext_{}_{}x{}d'.format(self.depth, self.cardinality, self.BOTTLENECK_OUT_CHANNEL)):
            with tf.variable_scope('preprocess_layers'):
                x = conv2d(x, self.FIRST_CONV_OUT_CHANNEL, kernel=3, stride=1, padding='SAME', name='conv')
                x = batch_normalization(x, name='bn', is_training=self.is_training)
                x = relu(x, name='relu')

            for idx, st in enumerate(self.structure):
                out_channel = self.cardinality * self.BOTTLENECK_OUT_CHANNEL * 2 ** idx

                if idx == 0:
                    first_stride = 1
                else:
                    first_stride = 2

                strides = [first_stride, *([1] * (st - 1))]

                for i, stride in zip(range(st), strides):
                    x = self.bottleneck(x, out_channel=out_channel, stride=stride, name='block_{}_{}'.format(idx, i))

            with tf.variable_scope('postprocess_layers'):
                if self.is_small:
                    x = global_avg_pooling(x, name='global_avg_pool')
                else:
                    x = avg_pooling(x, kernel=4, stride=4, name='avg_pool')

            with tf.variable_scope('classifier'):
                x = fully_connected(x, self.class_num, name='fully_connected')

            return x

if __name__ == "__main__":
    class_num = 10
    batch_size = None
    image = tf.placeholder(dtype=tf.float32, shape=[batch_size, 32, 32, 3])

    # model = SKResNeXt(29, class_num=class_num, is_small=False)

    # sk_out = model.sk_module(image, 4096, name='test_sk')
    # print(sk_out)
    # exit(0)

    for d in (29, ):
        model = SKResNeXt(d, class_num=class_num, is_small=False)
        output = model.build(image, is_training=False)

        print((batch_size, class_num), output.shape)

    from tools import print_net_info
    print_net_info()


#
# Dimensional and dimensionless parameter values, and scales
#
import pybamm
import pandas as pd
import os
import numbers
import warnings
from pprint import pformat
from collections import defaultdict

class ParameterValues:
    """
    The parameter values for a simulation.

    Note that this class does not inherit directly from the python dictionary class as
    this causes issues with saving and loading simulations.

    Parameters
    ----------
    values : dict or string
        Explicit set of parameters, or reference to a file of parameters
        If string, gets passed to read_parameters_csv to read a file.
    chemistry : dict
        Dict of strings for default chemistries. Must be of the form:
        {"base chemistry": base_chemistry,
        "cell": cell_properties_authorYear,
        "negative electrode": negative_electrode_chemistry_authorYear,
        "separator": separator_chemistry_authorYear,
        "positive electrode": positive_electrode_chemistry_authorYear,
        "electrolyte": electrolyte_chemistry_authorYear,
        "experiment": experimental_conditions_authorYear}.
        Then the negative electrode chemistry is loaded from the file
        inputs/parameters/base_chemistry/negative electrodes/
        negative_electrode_chemistry_authorYear, etc.
        Parameters in "cell" should include geometry and current collector properties.
        Parameters in "experiment" should include parameters relating to experimental
        conditions, such as initial conditions and currents.

    Examples
    --------
    >>> import pybamm
    >>> values = {"some parameter": 1, "another parameter": 2}
    >>> param = pybamm.ParameterValues(values)
    >>> param["some parameter"]
    1
    >>> file = "input/parameters/lithium-ion/cells/kokam_Marquis2019/parameters.csv"
    >>> values_path = pybamm.get_parameters_filepath(file)
    >>> param = pybamm.ParameterValues(values=values_path)
    >>> param["Negative current collector thickness [m]"]
    2.5e-05
    >>> param = pybamm.ParameterValues(chemistry=pybamm.parameter_sets.Marquis2019)
    >>> param["Reference temperature [K]"]
    298.15

    """

    def __init__(self, values=None, chemistry=None):
        self._dict_items = pybamm.FuzzyDict()
        # Must provide either values or chemistry, not both (nor neither)
        if values is not None and chemistry is not None:
            raise ValueError(
                "Only one of values and chemistry can be provided. To change parameters"
                " slightly from a chemistry, first load parameters with the chemistry"
                " (param = pybamm.ParameterValues(chemistry=...)) and then update with"
                " param.update({dict of values})."
            )
        if values is None and chemistry is None:
            raise ValueError("values and chemistry cannot both be None")
        # First load chemistry
        if chemistry is not None:
            self.update_from_chemistry(chemistry)
        # Then update with values dictionary or file
        if values is not None:
            # If base_parameters is a filename, load from that filename
            if isinstance(values, str):
                file_path = self.find_parameter(values)
                path = os.path.split(file_path)[0]
                values = self.read_parameters_csv(file_path)
            else:
                path = ""
            # Don't check parameter already exists when first creating it
            self.update(values, check_already_exists=False, path=path)

        # Initialise empty _processed_symbols dict (for caching)
        self._processed_symbols = {}
        self.parameter_events = []

    def __getitem__(self, key):
        return self._dict_items[key]

    def get(self, key, default=None):
        """Return item correspoonding to key if it exists, otherwise return default"""
        try:
            return self._dict_items[key]
        except KeyError:
            return default

    def __setitem__(self, key, value):
        """Call the update functionality when doing a setitem"""
        self.update({key: value})

    def __delitem__(self, key):
        del self._dict_items[key]

    def __repr__(self):
        return pformat(self._dict_items, width=1)

    def keys(self):
        """Get the keys of the dictionary"""
        return self._dict_items.keys()

    def values(self):
        """Get the values of the dictionary"""
        return self._dict_items.values()

    def items(self):
        """Get the items of the dictionary"""
        return self._dict_items.items()

    def copy(self):
        """Returns a copy of the parameter values. Makes sure to copy the internal
        dictionary."""
        return ParameterValues(values=self._dict_items.copy())

    def search(self, key, print_values=True):
        """
        Search dictionary for keys containing 'key'.

        See :meth:`pybamm.FuzzyDict.search()`.
        """
        return self._dict_items.search(key, print_values)

    def update_from_chemistry(self, chemistry):
        """
        Load standard set of components from a 'chemistry' dictionary
        """
        base_chemistry = chemistry["chemistry"]

        # Load each component name

        component_groups = [
            "cell",
            "negative electrode",
            "positive electrode",
            "separator",
            "electrolyte",
            "experiment",
        ]

        # add SEI parameters if provided
        if "sei" in chemistry:
            component_groups += ["sei"]

        if "anode" in chemistry.keys():
            if "negative electrode" in chemistry.keys():
                raise KeyError(
                    "both 'anode' and 'negative electrode' keys provided in the "
                    "chemistry. The 'anode' notation will be deprecated in the next "
                    "release so 'negative electrode' should be used instead."
                )
            else:
                chemistry["negative electrode"] = chemistry["anode"]
                warnings.warn(
                    "the 'anode' component notation will be deprecated in the next "
                    "release, as it has now been renamed to 'negative electrode'. "
                    "Simulation will continue passing the 'anode' component as "
                    "'negative electrode' (it might overwrite any existing definition "
                    "of the component).",
                    DeprecationWarning,
                )

        if "cathode" in chemistry.keys():
            if "positive electrode" in chemistry.keys():
                raise KeyError(
                    "both 'cathode' and 'positive electrode' keys provided in the "
                    "chemistry. The 'cathode' notation will be deprecated in the next "
                    "release so 'positive electrode' should be used instead."
                )
            else:
                chemistry["positive electrode"] = chemistry["cathode"]
                warnings.warn(
                    "the 'cathode' component notation will be deprecated in the next "
                    "release, as it has now been renamed to 'positive electrode'. "
                    "Simulation will continue passing the 'cathode' component as "
                    "'positive electrode' (it might overwrite any existing definition "
                    "of the component).",
                    DeprecationWarning,
                )

        for component_group in component_groups:
            # Make sure component is provided
            try:
                component = chemistry[component_group]
            except KeyError:
                raise KeyError(
                    "must provide '{}' parameters for {} chemistry".format(
                        component_group, base_chemistry
                    )
                )
            # Create path to component and load values
            component_path = os.path.join(
                base_chemistry, component_group.replace(" ", "_") + "s", component
            )
            file_path = self.find_parameter(
                os.path.join(component_path, "parameters.csv")
            )
            component_params = self.read_parameters_csv(file_path)

            # Update parameters, making sure to check any conflicts
            self.update(
                component_params,
                check_conflict=True,
                check_already_exists=False,
                path=os.path.dirname(file_path),
            )

        # register (list of) citations
        if "citation" in chemistry:
            citations = chemistry["citation"]
            if not isinstance(citations, list):
                citations = [citations]
            for citation in citations:
                pybamm.citations.register(citation)

    def read_parameters_csv(self, filename):
        """Reads parameters from csv file into dict.

        Parameters
        ----------
        filename : str
            The name of the csv file containing the parameters.

        Returns
        -------
        dict
            {name: value} pairs for the parameters.

        """
        df = pd.read_csv(filename, comment="#", skip_blank_lines=True)
        # Drop rows that are all NaN (seems to not work with skip_blank_lines)
        df.dropna(how="all", inplace=True)
        return {k: v for (k, v) in zip(df["Name [units]"], df["Value"])}

    def update(self, values, check_conflict=False, check_already_exists=True, path=""):
        """
        Update parameter dictionary, while also performing some basic checks.

        Parameters
        ----------
        values : dict
            Dictionary of parameter values to update parameter dictionary with
        check_conflict : bool, optional
            Whether to check that a parameter in `values` has not already been defined
            in the parameter class when updating it, and if so that its value does not
            change. This is set to True during initialisation, when parameters are
            combined from different sources, and is False by default otherwise
        check_already_exists : bool, optional
            Whether to check that a parameter in `values` already exists when trying to
            update it. This is to avoid cases where an intended change in the parameters
            is ignored due a typo in the parameter name, and is True by default but can
            be manually overridden.
        path : string, optional
            Path from which to load functions
        """
        # check parameter values
        self.check_parameter_values(values)
        # update
        for name, value in values.items():
            # check for conflicts
            if (
                check_conflict is True
                and name in self.keys()
                and not (self[name] == float(value) or self[name] == value)
            ):
                raise ValueError(
                    "parameter '{}' already defined with value '{}'".format(
                        name, self[name]
                    )
                )
            # check parameter already exists (for updating parameters)
            if check_already_exists is True:
                try:
                    self._dict_items[name]
                except KeyError as err:
                    raise KeyError(
                        "Cannot update parameter '{}' as it does not ".format(name)
                        + "have a default value. ({}). If you are ".format(err.args[0])
                        + "sure you want to update this parameter, use "
                        + "param.update({{name: value}}, check_already_exists=False)"
                    )
            # if no conflicts, update, loading functions and data if they are specified
            # Functions are flagged with the string "[function]"
            if isinstance(value, str):
                if value.startswith("[function]"):
                    loaded_value = pybamm.load_function(
                        os.path.join(path, value[10:] + ".py")
                    )
                    self._dict_items[name] = loaded_value
                    values[name] = loaded_value
                # Data is flagged with the string "[data]" or "[current data]"
                elif value.startswith("[current data]") or value.startswith("[data]"):
                    if value.startswith("[current data]"):
                        data_path = os.path.join(
                            pybamm.root_dir(), "pybamm", "input", "drive_cycles"
                        )
                        filename = os.path.join(data_path, value[14:] + ".csv")
                        function_name = value[14:]
                    else:
                        filename = os.path.join(path, value[6:] + ".csv")
                        function_name = value[6:]
                    filename = pybamm.get_parameters_filepath(filename)
                    data = pd.read_csv(
                        filename, comment="#", skip_blank_lines=True, header=None
                    ).to_numpy()
                    # Save name and data
                    self._dict_items[name] = (function_name, data)
                    values[name] = (function_name, data)
                elif value == "[input]":
                    self._dict_items[name] = pybamm.InputParameter(name)
                # Anything else should be a converted to a float
                else:
                    self._dict_items[name] = float(value)
                    values[name] = float(value)
            else:
                self._dict_items[name] = value
        # reset processed symbols
        self._processed_symbols = {}

    def check_parameter_values(self, values):
        # Make sure typical current is non-zero
        if "Typical current [A]" in values and values["Typical current [A]"] == 0:
            raise ValueError(
                "'Typical current [A]' cannot be zero. A possible alternative is to "
                "set 'Current function [A]' to `0` instead."
            )
        if "C-rate" in values:
            raise ValueError(
                "The 'C-rate' parameter has been deprecated, "
                "use 'Current function [A]' instead. The Nominal "
                "cell capacity can be accessed as 'Nominal cell "
                "capacity [A.h]', and used to calculate current from C-rate."
            )
        if "Cell capacity [A.h]" in values:
            if "Nominal cell capacity [A.h]" in values:
                raise ValueError(
                    "both 'Cell capacity [A.h]' and 'Nominal cell capacity [A.h]' "
                    "provided in values. The 'Cell capacity [A.h]' notation will be "
                    "deprecated in the next release so 'Nominal cell capacity [A.h]' "
                    "should be used instead."
                )
            else:
                values["Nominal cell capacity [A.h]"] = values["Cell capacity [A.h]"]
                warnings.warn(
                    "the 'Cell capacity [A.h]' notation will be "
                    "deprecated in the next release, as it has now been renamed "
                    "to 'Nominal cell capacity [A.h]'. Simulation will continue "
                    "passing the 'Cell capacity [A.h]' as 'Nominal cell "
                    "capacity [A.h]' (it might overwrite any existing definition "
                    "of the component)",
                    DeprecationWarning,
                )
        for param in values:
            if "surface area density" in param:
                raise ValueError(
                    "Parameters involving 'surface area density' have been renamed to "
                    "'surface area to volume ratio' ('{}' found)".format(param)
                )
            if "reaction rate" in param:
                raise ValueError(
                    "Parameters involving 'reaction rate' have been replaced with "
                    "'exchange-current density' ('{}' found)".format(param)
                )
        for param in values:
            if "particle distribution in x" in param:
                raise ValueError(
                    "The parameter '{}' has been deprecated".format(param)
                    + "The particle radius is now set as a function of x directly "
                    "instead of providing a reference value and a distribution."
                )
        for param in values:
            if "surface area to volume ratio distribution in x" in param:
                raise ValueError(
                    "The parameter '{}' has been deprecated".format(param)
                    + "The surface area to volume ratio is now set as a function "
                    "of x directly instead of providing a reference value and a "
                    "distribution."
                )

    def process_model(self, unprocessed_model, inplace=True):
        """Assign parameter values to a model.
        Currently inplace, could be changed to return a new model.

        Parameters
        ----------
        unprocessed_model : :class:`pybamm.BaseModel`
            Model to assign parameter values for
        inplace: bool, optional
            If True, replace the parameters in the model in place. Otherwise, return a
            new model with parameter values set. Default is True.

        Raises
        ------
        :class:`pybamm.ModelError`
            If an empty model is passed (`model.rhs = {}` and `model.algebraic = {}` and
            `model.variables = {}`)

        """
        pybamm.logger.info(
            "Start setting parameters for {}".format(unprocessed_model.name)
        )

        # set up inplace vs not inplace
        if inplace:
            # any changes to unprocessed_model attributes will change model attributes
            # since they point to the same object
            model = unprocessed_model
        else:
            # create a blank model of the same class
            model = unprocessed_model.new_empty_copy()

        if (
            len(unprocessed_model.rhs) == 0
            and len(unprocessed_model.algebraic) == 0
            and len(unprocessed_model.variables) == 0
        ):
            raise pybamm.ModelError("Cannot process parameters for empty model")

        new_rhs = {}
        for variable, equation in unprocessed_model.rhs.items():
            pybamm.logger.verbose(
                "Processing parameters for {!r} (rhs)".format(variable)
            )
            new_rhs[variable] = self.process_symbol(equation)
        model.rhs = new_rhs

        new_algebraic = {}
        for variable, equation in unprocessed_model.algebraic.items():
            pybamm.logger.verbose(
                "Processing parameters for {!r} (algebraic)".format(variable)
            )
            new_algebraic[variable] = self.process_symbol(equation)
        model.algebraic = new_algebraic

        new_initial_conditions = {}
        for variable, equation in unprocessed_model.initial_conditions.items():
            pybamm.logger.verbose(
                "Processing parameters for {!r} (initial conditions)".format(variable)
            )
            new_initial_conditions[variable] = self.process_symbol(equation)
        model.initial_conditions = new_initial_conditions

        model.boundary_conditions = self.process_boundary_conditions(unprocessed_model)

        new_variables = {}
        for variable, equation in unprocessed_model.variables.items():
            pybamm.logger.verbose(
                "Processing parameters for {!r} (variables)".format(variable)
            )
            new_variables[variable] = self.process_symbol(equation)
        model.variables = new_variables

        new_events = []
        for event in unprocessed_model.events:
            pybamm.logger.verbose(
                "Processing parameters for event '{}''".format(event.name)
            )
            new_events.append(
                pybamm.Event(
                    event.name, self.process_symbol(event.expression), event.event_type
                )
            )

        for event in self.parameter_events:
            pybamm.logger.verbose(
                "Processing parameters for event '{}''".format(event.name)
            )
            new_events.append(
                pybamm.Event(
                    event.name, self.process_symbol(event.expression), event.event_type
                )
            )

        model.events = new_events

        # Set external variables
        model.external_variables = [
            self.process_symbol(var) for var in unprocessed_model.external_variables
        ]

        # Process timescale
        model.timescale = self.process_symbol(unprocessed_model.timescale)

        # Process length scales
        new_length_scales = {}
        for domain, scale in unprocessed_model.length_scales.items():
            new_length_scales[domain] = self.process_symbol(scale)
        model.length_scales = new_length_scales

        pybamm.logger.info("Finish setting parameters for {}".format(model.name))

        return model

    def process_boundary_conditions(self, model):
        """
        Process boundary conditions for a model
        Boundary conditions are dictionaries {"left": left bc, "right": right bc}
        in general, but may be imposed on the tabs (or *not* on the tab) for a
        small number of variables, e.g. {"negative tab": neg. tab bc,
        "positive tab": pos. tab bc "no tab": no tab bc}.
        """
        new_boundary_conditions = {}
        sides = ["left", "right", "negative tab", "positive tab", "no tab"]
        for variable, bcs in model.boundary_conditions.items():
            processed_variable = self.process_symbol(variable)
            new_boundary_conditions[processed_variable] = {}
            for side in sides:
                try:
                    bc, typ = bcs[side]
                    pybamm.logger.verbose(
                        "Processing parameters for {!r} ({} bc)".format(variable, side)
                    )
                    processed_bc = (self.process_symbol(bc), typ)
                    new_boundary_conditions[processed_variable][side] = processed_bc
                except KeyError as err:
                    # don't raise error if the key error comes from the side not being
                    # found
                    if err.args[0] in side:
                        pass
                    # do raise error otherwise (e.g. can't process symbol)
                    else:
                        raise KeyError(err)

        return new_boundary_conditions

    def update_model(self, model, disc):
        raise NotImplementedError(
            """
            update_model functionality has been deprecated.
            Use pybamm.InputParameter to quickly change a parameter value instead
            """
        )

    def process_geometry(self, geometry):
        """
        Assign parameter values to a geometry (inplace).

        Parameters
        ----------
        geometry : dict
            Geometry specs to assign parameter values to
        """
        for domain in geometry:
            for spatial_variable, spatial_limits in geometry[domain].items():
                # process tab information if using 1 or 2D current collectors
                if spatial_variable == "tabs":
                    for tab, position_size in spatial_limits.items():
                        for position_size, sym in position_size.items():
                            geometry[domain]["tabs"][tab][
                                position_size
                            ] = self.process_symbol(sym)
                else:
                    for lim, sym in spatial_limits.items():
                        if isinstance(sym, pybamm.Symbol):
                            geometry[domain][spatial_variable][
                                lim
                            ] = self.process_symbol(sym)

    def process_symbol(self, symbol):
        """Walk through the symbol and replace any Parameter with a Value.
        If a symbol has already been processed, the stored value is returned.

        Parameters
        ----------
        symbol : :class:`pybamm.Symbol`
            Symbol or Expression tree to set parameters for

        Returns
        -------
        symbol : :class:`pybamm.Symbol`
            Symbol with Parameter instances replaced by Value

        """

        try:
            return self._processed_symbols[symbol.id]
        except KeyError:
            processed_symbol = self._process_symbol(symbol)
            self._processed_symbols[symbol.id] = processed_symbol

            return processed_symbol

    def _process_symbol(self, symbol):
        """ See :meth:`ParameterValues.process_symbol()`. """

        if isinstance(symbol, pybamm.Parameter):
            value = self[symbol.name]
            if isinstance(value, numbers.Number):
                # Scalar inherits name (for updating parameters) and domain (for
                # Broadcast)
                return pybamm.Scalar(value, name=symbol.name, domain=symbol.domain)
            elif isinstance(value, pybamm.Symbol):
                new_value = self.process_symbol(value)
                new_value.domain = symbol.domain
                return new_value
            else:
                raise TypeError("Cannot process parameter '{}'".format(value))

        elif isinstance(symbol, pybamm.FunctionParameter):
            new_children = []
            for child in symbol.children:
                if symbol.diff_variable is not None and any(
                    x.id == symbol.diff_variable.id for x in child.pre_order()
                ):
                    # Wrap with NotConstant to avoid simplification,
                    # which would stop symbolic diff from working properly
                    new_child = pybamm.NotConstant(child.new_copy())
                    new_children.append(self.process_symbol(new_child))
                else:
                    new_children.append(self.process_symbol(child))
            function_name = self[symbol.name]

            # Create Function or Interpolant or Scalar object
            if isinstance(function_name, tuple):
                # If function_name is a tuple then it should be (name, data) and we need
                # to create an Interpolant
                name, data = function_name
                function = pybamm.Interpolant(
                    data[:, 0], data[:, 1], *new_children, name=name
                )
                # Define event to catch extrapolation. In these events the sign is
                # important: it should be positive inside of the range and negative
                # outside of it
                self.parameter_events.append(
                    pybamm.Event(
                        "Interpolant {} lower bound".format(name),
                        pybamm.min(new_children[0] - min(data[:, 0])),
                        pybamm.EventType.INTERPOLANT_EXTRAPOLATION,
                    )
                )
                self.parameter_events.append(
                    pybamm.Event(
                        "Interpolant {} upper bound".format(name),
                        pybamm.min(max(data[:, 0]) - new_children[0]),
                        pybamm.EventType.INTERPOLANT_EXTRAPOLATION,
                    )
                )
            elif isinstance(function_name, numbers.Number):
                # If the "function" is provided is actually a scalar, return a Scalar
                # object instead of throwing an error.
                # Also use ones_like so that we get the right shapes
                function = pybamm.Scalar(
                    function_name, name=symbol.name
                ) * pybamm.ones_like(*new_children)
            elif (
                isinstance(function_name, pybamm.Symbol)
                and function_name.evaluates_to_number()
            ):
                # If the "function" provided is a pybamm scalar-like, use ones_like to
                # get the right shape
                # This also catches input parameters
                function = function_name * pybamm.ones_like(*new_children)
            elif callable(function_name):
                # otherwise evaluate the function to create a new PyBaMM object
                function = function_name(*new_children)
            elif isinstance(function_name, pybamm.Interpolant):
                function = function_name
            else:
                raise TypeError(
                    "Parameter provided for '{}' ".format(symbol.name)
                    + "is of the wrong type (should either be scalar-like or callable)"
                )
            # Differentiate if necessary
            if symbol.diff_variable is None:
                function_out = function
            else:
                # return differentiated function
                new_diff_variable = self.process_symbol(symbol.diff_variable)
                function_out = function.diff(new_diff_variable)
            # Convert possible float output to a pybamm scalar
            if isinstance(function_out, numbers.Number):
                return pybamm.Scalar(function_out)
            # Process again just to be sure
            return self.process_symbol(function_out)

        elif isinstance(symbol, pybamm.BinaryOperator):
            # process children
            new_left = self.process_symbol(symbol.left)
            new_right = self.process_symbol(symbol.right)
            # Special case for averages, which can appear as "integral of a broadcast"
            # divided by "integral of a broadcast"
            # this construction seems very specific but can appear often when averaging
            if (
                isinstance(symbol, pybamm.Division)
                # right is integral(Broadcast(1))
                and (
                    isinstance(new_right, pybamm.Integral)
                    and isinstance(new_right.child, pybamm.Broadcast)
                    and new_right.child.child.id == pybamm.Scalar(1).id
                )
                # left is integral
                and isinstance(new_left, pybamm.Integral)
            ):
                # left is integral(Broadcast)
                if (
                    isinstance(new_left.child, pybamm.Broadcast)
                    and new_left.child.child.domain == []
                ):
                    integrand = new_left.child
                    if integrand.auxiliary_domains == {}:
                        return integrand.orphans[0]
                    else:
                        domain = integrand.auxiliary_domains["secondary"]
                        if "tertiary" not in integrand.auxiliary_domains:
                            return pybamm.PrimaryBroadcast(integrand.orphans[0], domain)
                        else:
                            auxiliary_domains = {
                                "secondary": integrand.auxiliary_domains["tertiary"]
                            }
                            return pybamm.FullBroadcast(
                                integrand.orphans[0], domain, auxiliary_domains
                            )
                # left is "integral of concatenation of broadcasts"
                elif isinstance(new_left.child, pybamm.Concatenation) and all(
                    isinstance(child, pybamm.Broadcast)
                    for child in new_left.child.children
                ):
                    return self.process_symbol(pybamm.x_average(new_left.child))
            # make new symbol, ensure domain remains the same
            new_symbol = symbol._binary_new_copy(new_left, new_right)
            new_symbol.domain = symbol.domain
            return new_symbol

        # Unary operators
        elif isinstance(symbol, pybamm.UnaryOperator):
            new_child = self.process_symbol(symbol.child)
            new_symbol = symbol._unary_new_copy(new_child)
            # ensure domain remains the same
            new_symbol.domain = symbol.domain
            return new_symbol

        # Functions
        elif isinstance(symbol, pybamm.Function):
            new_children = [self.process_symbol(child) for child in symbol.children]
            return symbol._function_new_copy(new_children)

        # Concatenations
        elif isinstance(symbol, pybamm.Concatenation):
            new_children = [self.process_symbol(child) for child in symbol.children]
            return symbol._concatenation_new_copy(new_children)

        else:
            # Backup option: return new copy of the object
            try:
                return symbol.new_copy()
            except NotImplementedError:
                raise NotImplementedError(
                    "Cannot process parameters for symbol of type '{}'".format(
                        type(symbol)
                    )
                )

    def evaluate(self, symbol):
        """
        Process and evaluate a symbol.

        Parameters
        ----------
        symbol : :class:`pybamm.Symbol`
            Symbol or Expression tree to evaluate

        Returns
        -------
        number of array
            The evaluated symbol
        """
        processed_symbol = self.process_symbol(symbol)
        if processed_symbol.evaluates_to_constant_number():
            return processed_symbol.evaluate()
        else:
            raise ValueError("symbol must evaluate to a constant scalar")

    def _ipython_key_completions_(self):
        return list(self._dict_items.keys())

    def export_csv(self, filename):

        # process functions and data to output
        # like they appear in inputs csv files
        parameter_output = {}
        for key, val in self.items():
            if callable(val):
                val = "[function]" + val.__name__
            elif isinstance(val, tuple):
                val = "[data]" + val[0]
            parameter_output[key] = [val]

        df = pd.DataFrame(parameter_output)
        df = df.transpose()
        df.to_csv(filename, header=None)

    def print_parameters(self, parameters, output_file=None):
        """
        Return dictionary of evaluated parameters, and optionally print these evaluated
        parameters to an output file.
        For dimensionless parameters that depend on the C-rate, the value is given as a
        function of the C-rate (either x * Crate or x / Crate depending on the
        dependence)

        Parameters
        ----------
        parameters : class or dict containing :class:`pybamm.Parameter` objects
            Class or dictionary containing all the parameters to be evaluated
        output_file : string, optional
            The file to print parameters to. If None, the parameters are not printed,
            and this function simply acts as a test that all the parameters can be
            evaluated, and returns the dictionary of evaluated parameters.

        Returns
        -------
        evaluated_parameters : defaultdict
            The evaluated parameters, for further processing if needed

        Notes
        -----
        A C-rate of 1 C is the current required to fully discharge the battery in 1
        hour, 2 C is current to discharge the battery in 0.5 hours, etc
        """
        # Set list of attributes to ignore, for when we are evaluating parameters from
        # a class of parameters
        ignore = [
            "__name__",
            "__doc__",
            "__package__",
            "__loader__",
            "__spec__",
            "__file__",
            "__cached__",
            "__builtins__",
            "absolute_import",
            "division",
            "print_function",
            "unicode_literals",
            "pybamm",
            "constants",
            "np",
            "geo",
            "elec",
            "therm",
        ]

        # If 'parameters' is a class, extract the dict
        if not isinstance(parameters, dict):
            parameters = {
                k: v for k, v in parameters.__dict__.items() if k not in ignore
            }

        evaluated_parameters = defaultdict(list)
        # Calculate parameters for each C-rate
        for Crate in [1, 10]:
            # Update Crate
            capacity = self.get("Nominal cell capacity [A.h]")
            if capacity is not None:
                self.update(
                    {"Current function [A]": Crate * capacity},
                    check_already_exists=False,
                )
            for name, symbol in parameters.items():
                if not callable(symbol):
                    proc_symbol = self.process_symbol(symbol)
                    if not (
                        callable(proc_symbol)
                        or proc_symbol.has_symbol_of_classes(
                            (pybamm.Concatenation, pybamm.Broadcast)
                        )
                    ):
                        evaluated_parameters[name].append(proc_symbol.evaluate(t=0))

        # Calculate C-dependence of the parameters based on the difference between the
        # value at 1C and the value at C / 10
        for name, values in evaluated_parameters.items():
            if values[1] == 0 or abs(values[0] / values[1] - 1) < 1e-10:
                C_dependence = ""
            elif abs(values[0] / values[1] - 10) < 1e-10:
                C_dependence = " * Crate"
            elif abs(values[0] / values[1] - 0.1) < 1e-10:
                C_dependence = " / Crate"
            evaluated_parameters[name] = (values[0], C_dependence)
        # Print the evaluated_parameters dict to output_file
        if output_file:
            self.print_evaluated_parameters(evaluated_parameters, output_file)

        return evaluated_parameters

    def print_evaluated_parameters(self, evaluated_parameters, output_file):
        """
        Print a dictionary of evaluated parameters to an output file

        Parameters
        ----------
        evaluated_parameters : defaultdict
            The evaluated parameters, for further processing if needed
        output_file : string, optional
            The file to print parameters to. If None, the parameters are not printed,
            and this function simply acts as a test that all the parameters can be
            evaluated

        """
        # Get column width for pretty printing
        column_width = max(len(name) for name in evaluated_parameters.keys())
        s = "{{:>{}}}".format(column_width)
        with open(output_file, "w") as file:
            for name, (value, C_dependence) in sorted(evaluated_parameters.items()):
                if 0.001 < abs(value) < 1000:
                    file.write(
                        (s + " : {:10.4g}{!s}\n").format(name, value, C_dependence)
                    )
                else:
                    file.write(
                        (s + " : {:10.3E}{!s}\n").format(name, value, C_dependence)
                    )

    @staticmethod
    def find_parameter(path):
        """Look for parameter file in the different locations
        in PARAMETER_PATH
        """
        for location in pybamm.PARAMETER_PATH:
            trial_path = os.path.join(location, path)
            if os.path.isfile(trial_path):
                return trial_path
        raise FileNotFoundError("Could not find parameter {}".format(path))


import json
import traceback
import io
import pickle
import copy
from git import Repo
from . import app_logger
from web_app.auxiliary import exceptions

def load_model(model_spec):
    """Given a model and its version, load it"""

    #models_repo = r"/Users/davidlaredorazo/Documents/Projects/Rappi Challenge/models_and_data"
    models_repo = r"/models_and_data"

    model = None

    #Try to open repository
    try:
        repo = Repo(models_repo)
    except Exception as e:
        app_logger.error('Could not open repository')
        app_logger.error(traceback.format_exc())
        raise exceptions.FileError('Could not open repository')

    #Attempt to load models
    try:
        if not model_spec['model_tag']:
            raise exceptions.UnspecifiedModel('Model not specified')

        if model_spec['model_version']:
            commit = repo.commit(model_spec['model_version'])

            target_file = commit.tree / ('models/deploy/' + model_spec['model_tag'] + '.pkl')

            print(target_file)

            with io.BytesIO(target_file.data_stream.read()) as f:
                model = pickle.load(f)
        else:
            model = pickle.load(open(models_repo + '/models/deploy/' + model_spec['model_tag'] + '.pkl', 'rb'))
    except Exception as e:
        app_logger.error('Could not load model')
        app_logger.error(traceback.format_exc())
        raise exceptions.UnspecifiedModel('Could not load model')

    return model

def load_ml_models(app_root):
    """Load all the models specified in the models_list"""

    models_list = None
    models = {}
    model = None
    print(app_root)

    try:

        app_logger.error("Loading models from")
        app_logger.error(app_root / 'models_list.json')

        with open(app_root / 'models_list.json', 'r') as fp:
            models_list = json.load(fp)

        app_logger.error(models_list)

    except Exception as e:

        app_logger.error("Could not open models list file.")
        app_logger.error(traceback.format_exc())
        print("Could not open models list file. Check app_log file")

    for key in models_list:

        model_spec = models_list[key]
        model = load_model(model_spec)
        models[model_spec['model_tag'] + '/' +
               (model_spec['model_version'] if model_spec['model_version'] else 'latest')] = copy.deepcopy(model)

    return models


from django.test import TestCase,Client
from .models import Airport
from .models import Flight

# Create your tests here.

class ModelsTestCase(TestCase):
    def setUp(self):

        a1 =  Airport.objects.create(code="AAA",city="City A")
        a2 = Airport.objects.create(code= "BBB", city = "City B")

        Flight.objects.create(origin=a1,destination=a2,duration=100)
        Flight.objects.create(origin=a1,destination=a1,duration=200)
        Flight.objects.create(origin=a1,destination=a2,duration=-100)

    def test_departure_count(self):
        a = Airport.objects.get(code ="AAA")
        self.assertEqual(a.departures.count(),3)

    def test_arrival_count(self):
        a = Airport.objects.get(code ="AAA")
        self.assertEqual(a.arrivals.count(),1)

    def test_valid_flight(self):
        a1 = Airport.objects.get(code ="AAA")
        a2 = Airport.objects.get(code ="BBB")
        f = Flight.objects.get(origin=a1,destination= a2,duration=100)
        self.assertTrue(f.is_valid_flight())

    def test_invalid_flight_destination(self):
        a1 = Airport.objects.get(code ="AAA")
        f = Flight.objects.get(origin=a1,destination= a1)
        self.assertFalse(f.is_valid_flight())

    def test_invalid_flight_duration(self):
        a1 = Airport.objects.get(code ="AAA")
        a2 = Airport.objects.get(code ="BBB")
        f = Flight.objects.get(origin=a1,destination= a2,duration=-100)
        self.assertFalse(f.is_valid_flight())

    def test_index(self):
        c = Client()
        response = c.get("/")
        self.assertEqual(response.status_code,200)
        self.assertEqual(response.context["flights"].count(),3)

    def test_valid_flight_page(self):
        a1 = Airport.objects.get(code = "AAA")
        f =  Flight.objects.get(origin = a1, destination = a1)
        c = Client()
        response = c.get(f"/{f.id}")
        self.assertEqual(response.status_code,200)

   #def test_invalid_flight(self):
    #    max_id = Flight.objects.all().aggregate(Max("id")["id_max"])
     #   c = Client()
      #  response = c.get(f"/{max_id +1}")
       # self.assertEqual(response.status_code,404)


# coding: utf-8

# In[ ]:

import datetime
from sklearn.feature_selection import chi2, SelectPercentile
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_extraction.text import CountVectorizer
from scipy import sparse
import lightgbm as lgb
import warnings
import time
import pandas as pd
import numpy as np
import os

# In[ ]:

train = pd.read_hdf('../data/train_data2.hdf', key='xunfei')

test = pd.read_hdf('../data/test_data2.hdf', key='xunfei')
data = pd.concat([train, test], axis=0, ignore_index=True)

# In[ ]:

pd.set_option('display.max_columns',100)
data['time']=(data['timee']-data['timee'].min())/(data['timee'].max()-data['timee'].min())

# In[ ]:

ad_cate_feature=['adid','advert_id','orderid','campaign_id','creative_id','creative_tp_dnf','creative_type',
       'creative_is_jump','creative_is_download','creative_has_deeplink','advert_industry_inner0','advert_industry_inner1']
media_cate_feature=['app_cate_id','f_channel','app_id','inner_slot_id']
content_cate_feature=['city', 'carrier', 'province', 'nnt', 'devtype', 'osv', 'make', 'model']

all_data=data.copy()

need=list(all_data.index)

# 循环读取
import gc
from tqdm import tqdm#'unsparse','timesparse',,'expsparse1','expsparse2','dupsparse','innersparse1','dupsparse1'
filelist=['expsparsex','expsparse1','innersparse1','appearsparses']#'svdsparsenew',
start=1
tail=0
##all_df=sparse.csr_matrix((len(all_data),0))
maxrec={}
for f in filelist:

    ttf=pd.read_hdf( '../data/'+f+'.hdf').astype(np.float32).iloc[need,:]
    orgshape=ttf.shape[1]

    '''use=pd.read_hdf('../data/'+f+'_catvip'+'.hdf',key='xunfei')['catvip'].values

    ttf=ttf.iloc[:,use]
    maxrec[f]=use#ttf.shape[1]'''
    if (start==1):
        all_df=ttf.copy()
        start=0
    else:
        all_df=pd.concat([all_df,ttf],axis=1)
    print(f,orgshape,(ttf.shape[1])/orgshape)
    del ttf
    gc.collect()

# In[ ]:

from tqdm import tqdm
num_feature = ['creative_width', 'creative_height', 'hour','user_tags_len']#,'time'
ttmp=data[num_feature].values.astype(np.int32)[need,:]
for col in tqdm(range(ttmp.shape[1])):
    all_df['num'+str(col)]=ttmp[:,col]

del ttmp
gc.collect()

# In[ ]:

# 默认加载 如果 增加了cate类别特征 请改成false重新生成
import os
if os.path.exists('../feature/base_train_csr.npz') and True:
    print('load_csr---------')
    base_train_csr = sparse.load_npz( '../feature/base_train_csr.npz').tocsr().astype('bool')
    base_predict_csr = sparse.load_npz('../feature/base_predict_csr.npz').tocsr().astype('bool')
else:
    base_train_csr = sparse.csr_matrix((len(train), 0))
    base_predict_csr = sparse.csr_matrix((len(predict_x), 0))

    enc = OneHotEncoder()
    for feature in cate_feature:
        enc.fit(data[feature].values.reshape(-1, 1))
        base_train_csr = sparse.hstack((base_train_csr, enc.transform(train_x[feature].values.reshape(-1, 1))), 'csr',
                                       'bool')
        base_predict_csr = sparse.hstack((base_predict_csr, enc.transform(predict[feature].values.reshape(-1, 1))),
                                         'csr',
                                         'bool')
    print('one-hot prepared !')

    cv = CountVectorizer(min_df=10)
    for feature in ['user_tagss']:
        data[feature] = data[feature].astype(str)
        cv.fit(data[feature])
        base_train_csr = sparse.hstack((base_train_csr, cv.transform(train_x[feature].astype(str))), 'csr', 'bool')
        base_predict_csr = sparse.hstack((base_predict_csr, cv.transform(predict_x[feature].astype(str))), 'csr',
                                         'bool')
    print('cv prepared !')

    sparse.save_npz( '../feature/base_train_csr.npz', base_train_csr)
    sparse.save_npz('../feature/base_predict_csr.npz', base_predict_csr)

# In[ ]:

#catvip=pd.read_hdf('../data/catvip.hdf', key='xunfei')['catvip'].values
allbase=sparse.vstack((base_train_csr,base_predict_csr),'csr')
#allbase=allbase[need,:][:,catvip]
#allbase=allbase.toarray()

# In[ ]:

all_df=sparse.hstack((allbase,all_df),'csr')

# In[ ]:

train_x =all_df[:-len(test)]
train_y = data[data.target != -1].target.values

# In[ ]:

test_x=all_df[-len(test):]

# In[ ]:

import gc
del allbase
gc.collect()
del all_df
gc.collect()
del data
gc.collect()

# In[ ]:

del all_data
gc.collect()

# In[ ]:

import datetime
import lightgbm as lgb
import gc
from scipy.sparse import csr_matrix
from sklearn.model_selection import KFold, StratifiedKFold
# sklearn风格lightgbm
import random
fea_rec=[]
fea_vip=[]
sub_preds = pd.DataFrame()

oof_predss = np.zeros(train_x.shape[0])
#argsDict={'bagging_fraction': 0.7441379310344828, 'bagging_freq': 2, 'feature_fraction': 0.8489655172413793, 'learning_rate': 0.07241379310344827, 'max_bin': 268, 'max_depth': -1, 'min_child_weight': 1.3684210526315788, 'min_data_in_bin': 10, 'min_split_gain': 0.21052631578947367, 'num_boost_round': 5000, 'num_leaves': 50, 'rands': 396, 'reg_alpha': 1.8421052631578947, 'reg_lambda': 7.894736842105263, 'scale_pos_weight': 1.0}

argsDict={'bagging_fraction': 0.98, 'bagging_freq': 8, 'feature_fraction': 0.9027586206896552, 'learning_rate': 0.052631578947368425, 'max_bin': 260, 'max_depth': -1, 'min_child_weight': 0.1724137931034483, 'min_data_in_bin': 7, 'min_split_gain': 0.1, 'num_boost_round': 5000, 'num_leaves': 56, 'rands': 152, 'reg_alpha': 6.842105263157895, 'reg_lambda': 13.974358974358974, 'scale_pos_weight': 1.0}

LGB=lgb.LGBMClassifier(
        num_leaves=argsDict["num_leaves"],
        max_depth=argsDict["max_depth"],
        learning_rate=argsDict["learning_rate"],
        n_estimators=argsDict['num_boost_round'],
         min_split_gain=argsDict["min_split_gain"],
        #min_child_samples=argsDict["min_data_in_leaf"],
        min_child_weight=argsDict["min_child_weight"],
        subsample=argsDict["bagging_fraction"],
        subsample_freq=argsDict["bagging_freq"],
        colsample_bytree=argsDict["feature_fraction"],
        reg_alpha=argsDict["reg_alpha"],
        reg_lambda=argsDict["reg_lambda"],
        scale_pos_weight=argsDict["scale_pos_weight"],

        is_training_metric= True,
        boosting_type='gbdt',
        metric='binary_logloss',
        n_jobs=9,
        #n_threads=10,
        seed=argsDict["rands"],
        drop_seed=argsDict["rands"],
        bagging_seed=argsDict["rands"],
        feature_fraction_seed=argsDict["rands"],
        random_state=argsDict["rands"],

        max_bin=argsDict["max_bin"],
        min_data_in_bin=argsDict["min_data_in_bin"],
    )
rands=random.randint(0,100)
skf = StratifiedKFold(n_splits=20, shuffle=True, random_state=argsDict["rands"])
tmprmselist=[]
treerec=[]
for ii,(train_index, test_index) in enumerate(skf.split(train_x, train_y)):

    x_train, x_val = train_x[train_index], train_x[test_index]
    y_train, y_val = train_y[train_index], train_y[test_index]
    print('fold:',ii)
    LGB.fit(x_train,y_train.reshape(-1,),eval_set=(x_val,y_val),eval_metric='logloss',early_stopping_rounds=100,verbose=100)
    tmplist=list(LGB.predict_proba(x_val)[:,1])
    oof_predss[test_index] =tmplist
        #print(LGB.evals_result_)
    tmprmse =np.mean(LGB.evals_result_['valid_0']['binary_logloss'])#np.sqrt(mean_squared_error(y_val, tmplist))
    print('best_itor:',LGB.best_iteration_,'    logloss:',tmprmse)
    treerec.append(LGB.best_iteration_)
    tmprmselist.append(tmprmse)

    sub_preds['estimators'+str(ii)]=(LGB.predict_proba(test_x,num_iteration=LGB.best_iteration_))[:,1]

    importance_dict={}
    for col,val in zip(range(train_x.shape[1]),LGB.feature_importances_):
        importance_dict[col]=val
    ser=pd.Series(importance_dict).sort_values(ascending=False)
    fea_vip+=list(ser[ser>3].index)
    del x_train
    gc.collect()
    del x_val
    gc.collect()

# In[ ]:

sub=test[['instance_id']]
sub['predicted_score']=(sub_preds.mean(axis=1)).values
org_test=pd.read_table('../data/round2_iflyad_test_feature.txt')
sub=org_test[['instance_id']].merge(sub,on='instance_id',how='left')
sub.to_csv('../subs/10151.csv',index=None)

# In[ ]:

oof_df=pd.DataFrame()
oof_df['instance_id']=train['instance_id'].values
oof_df['oof']=oof_predss.reshape(-1,)
oof_df.to_hdf('../oof/10151.csv',key='xunfei')