# Copyright The IETF Trust 2015, All Rights Reserved

from django.db import models
from django.template import Template, Context

from email.utils import parseaddr
from ietf.utils.mail import formataddr
from ietf.person.models import Email

import debug                            # pyflakes:ignore

from ietf.group.models import Role

def clean_duplicates(addrlist):
    address_info = {}
    for a in addrlist:
        (name,addr) = parseaddr(a)
        # This collapses duplicate addresses to one, using (arbitrarily) the
        # name from the last one:
        address_info[addr] = (name, a)
    addresses = []
    for addr, info in address_info.items():
        name, a = info
        if (name,addr)==('',''):
            addresses.append(a)
        elif name:
            addresses.append(formataddr((name,addr)))
        else:
            addresses.append(addr)
    return addresses

class MailTrigger(models.Model):
    slug = models.CharField(max_length=32, primary_key=True)
    desc = models.TextField(blank=True)
    to   = models.ManyToManyField('Recipient', blank=True, related_name='used_in_to')
    cc   = models.ManyToManyField('Recipient', blank=True, related_name='used_in_cc')

    class Meta:
        ordering = ["slug"]

    def __unicode__(self):
        return self.slug

class Recipient(models.Model):
    slug = models.CharField(max_length=32, primary_key=True)
    desc = models.TextField(blank=True)
    template = models.TextField(null=True, blank=True)

    class Meta:
        ordering = ["slug"]

    def __unicode__(self):
        return self.slug

    def gather(self, **kwargs):
        retval = []
        if hasattr(self,'gather_%s'%self.slug):
            retval.extend(eval('self.gather_%s(**kwargs)'%self.slug))
        if self.template:
            rendering = Template('{%% autoescape off %%}%s{%% endautoescape %%}'%self.template).render(Context(kwargs))
            if rendering:
                retval.extend([x.strip() for x in rendering.split(',')])

        return clean_duplicates(retval)

    def gather_doc_group_chairs(self, **kwargs):
        addrs = []
        if 'doc' in kwargs:
            doc=kwargs['doc']
            if doc.group and doc.group.type.slug in ['wg','rg','ag',]:
                addrs.append('%s-chairs@ietf.org'%doc.group.acronym)
        return addrs

    def gather_doc_group_delegates(self, **kwargs):
        addrs = []
        if 'doc' in kwargs:
            doc=kwargs['doc']
            if doc.group and doc.group.type.slug in ['wg','rg','ag',]:
                addrs.extend(doc.group.role_set.filter(name='delegate').values_list('email__address',flat=True))
        return addrs

    def gather_doc_group_mail_list(self, **kwargs):
        addrs = []
        if 'doc' in kwargs:
            doc=kwargs['doc']
            if doc.group.type.slug in ['wg','rg','ag',]:
                if doc.group.list_email:
                    addrs.append(doc.group.list_email)
        return addrs

    def gather_doc_affecteddoc_authors(self, **kwargs):
        addrs = []
        if 'doc' in kwargs:
            for reldoc in kwargs['doc'].related_that_doc(('conflrev','tohist','tois','tops')):
                addrs.extend(Recipient.objects.get(slug='doc_authors').gather(**{'doc':reldoc.document}))
        return addrs

    def gather_doc_affecteddoc_group_chairs(self, **kwargs):
        addrs = []
        if 'doc' in kwargs:
            for reldoc in kwargs['doc'].related_that_doc(('conflrev','tohist','tois','tops')):
                addrs.extend(Recipient.objects.get(slug='doc_group_chairs').gather(**{'doc':reldoc.document}))
        return addrs

    def gather_doc_affecteddoc_notify(self, **kwargs):
        addrs = []
        if 'doc' in kwargs:
            for reldoc in kwargs['doc'].related_that_doc(('conflrev','tohist','tois','tops')):
                addrs.extend(Recipient.objects.get(slug='doc_notify').gather(**{'doc':reldoc.document}))
        return addrs

    def gather_conflict_review_stream_manager(self, **kwargs):
        addrs = []
        if 'doc' in kwargs:
            for reldoc in kwargs['doc'].related_that_doc(('conflrev',)):
                addrs.extend(Recipient.objects.get(slug='doc_stream_manager').gather(**{'doc':reldoc.document}))
        return addrs

    def gather_conflict_review_steering_group(self,**kwargs):
        addrs = []
        if 'doc' in kwargs:
            for reldoc in kwargs['doc'].related_that_doc(('conflrev',)):
                if reldoc.document.stream_id=='irtf':
                    addrs.append('"Internet Research Steering Group" <irsg@irtf.org>')
        return addrs

    def gather_group_steering_group(self,**kwargs):
        addrs = []
        sg_map = dict( wg='"The IESG" <iesg@ietf.org>', rg='"Internet Research Steering Group" <irsg@irtf.org>' )
        if 'group' in kwargs and kwargs['group'].type_id in sg_map:
            addrs.append(sg_map[kwargs['group'].type_id])
        return addrs

    def gather_stream_managers(self, **kwargs):
        addrs = []
        manager_map = dict(ise  = '<rfc-ise@rfc-editor.org>',
                           irtf = '<irtf-chair@irtf.org>',
                           ietf = '<iesg@ietf.org>',
                           iab  = '<iab-chair@iab.org>')
        if 'streams' in kwargs:
            for stream in kwargs['streams']:
                if stream in manager_map:
                    addrs.append(manager_map[stream])
        return addrs

    def gather_doc_stream_manager(self, **kwargs):
        addrs = []
        if 'doc' in kwargs:
            addrs.extend(Recipient.objects.get(slug='stream_managers').gather(**{'streams':[kwargs['doc'].stream_id]}))
        return addrs

    def gather_doc_non_ietf_stream_manager(self, **kwargs):
        addrs = []
        if 'doc' in kwargs:
            doc = kwargs['doc']
            if doc.stream_id and doc.stream_id != 'ietf':
                addrs.extend(Recipient.objects.get(slug='stream_managers').gather(**{'streams':[doc.stream_id,]}))
        return addrs

    def gather_group_responsible_directors(self, **kwargs):
        addrs = []
        if 'group' in kwargs:
            group = kwargs['group']
            if not group.acronym=='none':
                addrs.extend(group.role_set.filter(name='ad').values_list('email__address',flat=True))
            if group.type_id=='rg':
                addrs.extend(Recipient.objects.get(slug='stream_managers').gather(**{'streams':['irtf']}))
        return addrs

    def gather_group_secretaries(self, **kwargs):
        addrs = []
        if 'group' in kwargs:
            group = kwargs['group']
            if not group.acronym=='none':
                addrs.extend(group.role_set.filter(name='secr').values_list('email__address',flat=True))
        return addrs

    def gather_doc_group_responsible_directors(self, **kwargs):
        addrs = []
        if 'doc' in kwargs:
            group = kwargs['doc'].group
            if group and not group.acronym=='none':
                addrs.extend(Recipient.objects.get(slug='group_responsible_directors').gather(**{'group':group}))
        return addrs

    def gather_submission_authors(self, **kwargs):
        """
        Returns a list of name and email, e.g.: [ 'Ano Nymous <ano@nymous.org>' ]
        Is intended for display use, not in email context.
        """
        addrs = []
        if 'submission' in kwargs:
            submission = kwargs['submission']
            addrs.extend(["%s <%s>" % (author["name"], author["email"]) for author in submission.authors if author.get("email")])
        return addrs

    def gather_submission_group_chairs(self, **kwargs):
        addrs = []
        if 'submission' in kwargs:
            submission = kwargs['submission']
            if submission.group:
                addrs.extend(Recipient.objects.get(slug='group_chairs').gather(**{'group':submission.group}))
        return addrs

    def gather_submission_confirmers(self, **kwargs):
        """If a submitted document is revising an existing document, the confirmers
           are the authors of that existing document, and the chairs if the document is
           a working group document and the author list has changed. Otherwise, the confirmers
           are the authors and submitter of the submitted document."""

        addrs=[]
        if 'submission' in kwargs:
            submission = kwargs['submission']
            doc=submission.existing_document()
            if doc:
                old_authors = [ author for author in doc.documentauthor_set.all() if author.email ]

                addrs.extend([ author.formatted_email() for author in old_authors])

                old_author_email_set = set(author.email.address for author in old_authors)
                new_author_email_set = set(author["email"] for author in submission.authors if author.get("email"))

                if doc.group and old_author_email_set != new_author_email_set:
                    if doc.group.type_id in ['wg','rg','ag']:
                        addrs.extend(Recipient.objects.get(slug='group_chairs').gather(**{'group':doc.group}))
                    elif doc.group.type_id in ['area']:
                        addrs.extend(Recipient.objects.get(slug='group_responsible_directors').gather(**{'group':doc.group}))
                    else:
                        pass
                    if doc.stream_id and doc.stream_id not in ['ietf']:
                        addrs.extend(Recipient.objects.get(slug='stream_managers').gather(**{'streams':[doc.stream_id]}))
            else:
                # This is a bit roundabout, but we do it to get consistent and unicode-compliant
                # email names for known persons, without relying on the name parsed from the
                # draft (which might be ascii, also for persons with non-ascii names)
                emails = [ Email.objects.filter(address=author['email']).first() or author for author in submission.authors if author.get('email') ]
                addrs.extend([ e.formatted_email() if isinstance(e, Email) else formataddr((e["name"], e["email"])) for e in emails ] )
                submitter_email = submission.submitter_parsed()["email"]
                if submitter_email and not submitter_email in [ parseaddr(a)[1] for a in addrs ]:
                    addrs.append(submission.submitter)
        return addrs

    def gather_submission_group_mail_list(self, **kwargs):
        addrs=[]
        if 'submission' in kwargs:
            submission = kwargs['submission']
            if submission.group:
                addrs.extend(Recipient.objects.get(slug='group_mail_list').gather(**{'group':submission.group}))
        return addrs

    def gather_rfc_editor_if_doc_in_queue(self, **kwargs):
        addrs=[]
        if 'doc' in kwargs:
            doc = kwargs['doc']
            if doc.get_state_slug("draft-rfceditor") is not None:
                addrs.extend(Recipient.objects.get(slug='rfc_editor').gather(**{}))
        return addrs

    def gather_doc_discussing_ads(self, **kwargs):
        addrs=[]
        if 'doc' in kwargs:
            doc = kwargs['doc']
            active_ballot = doc.active_ballot()
            if active_ballot:
                for ad, pos in active_ballot.active_ad_positions().iteritems():
                    if pos and pos.pos_id == "discuss":
                        addrs.append(ad.role_email("ad").address)
        return addrs

    def gather_ipr_updatedipr_contacts(self, **kwargs):
        addrs=[]
        if 'ipr' in kwargs:
            ipr = kwargs['ipr']
            for rel in ipr.updates:
                if rel.target.submitter_email:
                    addrs.append(rel.target.submitter_email)
                elif hasattr(rel.target,'ietfer_email') and rel.target.ietfer_email:
                    addrs.append(rel.target.ietfer_email)
        return addrs

    def gather_ipr_updatedipr_holders(self, **kwargs):
        addrs=[]
        if 'ipr' in kwargs:
            ipr = kwargs['ipr']
            for disc in ipr.recursively_updates():
                if hasattr(ipr,'holder_contact_email') and ipr.holder_contact_email:
                    addrs.append(ipr.holder_contact_email)
        return addrs

    def gather_doc_ipr_group_or_ad(self, **kwargs):
        """A document's group email list if the document is a group document,
           otherwise, the document's AD if the document is active, otherwise
           the IETF chair"""
        addrs=[]
        if 'doc' in kwargs:
            doc=kwargs['doc']
            if doc.group and doc.group.acronym == 'none':
                if doc.ad and doc.get_state_slug('draft')=='active':
                    addrs.extend(Recipient.objects.get(slug='doc_ad').gather(**kwargs))
                else:
                    addrs.extend(Role.objects.filter(group__acronym='gen',name='ad').values_list('email__address',flat=True))
            else:
                addrs.extend(Recipient.objects.get(slug='doc_group_mail_list').gather(**kwargs))
        return addrs

    def gather_liaison_manager(self, **kwargs):
        addrs=[]
        if 'group' in kwargs:
            group=kwargs['group']
            addrs.extend(group.role_set.filter(name='liaiman').values_list('email__address',flat=True))
        return addrs

    def gather_session_requester(self, **kwargs):
        addrs=[]
        if 'session' in kwargs:
            session = kwargs['session']
            addrs.append(session.requested_by.role_email('chair').address)
        return addrs

    def gather_review_team_ads(self, **kwargs):
        addrs=[]
        if 'review_req' in kwargs:
            review_req = kwargs['review_req']
            if review_req.team.parent:
                for role in review_req.team.parent.role_set.filter(name='ad'):
                    addrs.append(role.email.address)
        return addrs


from yaml import load
from exp.pendulum_experiment import PendulumExperiment
from control.random_policy import RandomPolicy
from model.elbo_loss import ELBOLoss
from model.variational_koopman import VariationalKoopman
from exp.dataset import TrajectoryDataset
import argparse
import utils
import torch.nn as nn
import torch.optim as optim
import torch
import numpy as np

import pickle

ExperimentList = {'Pendulum': PendulumExperiment}
ActivationList = {'ELU': nn.ELU, 'ReLU': nn.ReLU, 'Sigmoid': nn.Sigmoid, 'Tanh': nn.Tanh}
LossList = {'ELBOLoss': ELBOLoss}
OptimList = {'Adam': optim.Adam, 'RMSProp': optim.RMSprop, 'SGD': optim.SGD}
PolicyList = {'RandomPolicy': RandomPolicy}

def load_TF_weights(net, net_weights):
    param = net_weights['values']

    net.feat_extractor[0].weight.data = torch.from_numpy(param[0]).to(torch.float).t()
    net.feat_extractor[0].bias.data = torch.from_numpy(param[1]).to(torch.float)
    net.feat_extractor[2].weight.data = torch.from_numpy(param[2]).to(torch.float).t()
    net.feat_extractor[2].bias.data = torch.from_numpy(param[3]).to(torch.float)

    net.temporal_encoder_lstm.LSTM_f.cell.gates_mlp.weight.data = torch.from_numpy(param[4]).to(torch.float).t()
    net.temporal_encoder_lstm.LSTM_f.cell.gates_mlp.bias.data = torch.from_numpy(param[5]).to(torch.float)
    net.temporal_encoder_lstm.LSTM_b.cell.gates_mlp.weight.data = torch.from_numpy(param[6]).to(torch.float).t()
    net.temporal_encoder_lstm.LSTM_b.cell.gates_mlp.bias.data = torch.from_numpy(param[7]).to(torch.float)

    net.temporal_encoder_mlp[0].weight.data = torch.from_numpy(param[8]).to(torch.float).t()
    net.temporal_encoder_mlp[0].bias.data = torch.from_numpy(param[9]).to(torch.float)
    net.temporal_encoder_mlp[2].weight.data = torch.from_numpy(param[10]).to(torch.float).t()
    net.temporal_encoder_mlp[2].bias.data = torch.from_numpy(param[11]).to(torch.float)

    net.init_basis_mlp[0].weight.data = torch.from_numpy(param[12]).to(torch.float).t()
    net.init_basis_mlp[0].bias.data = torch.from_numpy(param[13]).to(torch.float)
    net.init_basis_mlp[2].weight.data = torch.from_numpy(param[14]).to(torch.float).t()
    net.init_basis_mlp[2].bias.data = torch.from_numpy(param[15]).to(torch.float)

    net.basis_encoder_gru.gates_mlp.weight.data = torch.from_numpy(param[24]).to(torch.float).t()
    net.basis_encoder_gru.gates_mlp.bias.data = torch.from_numpy(param[25]).to(torch.float)
    net.basis_encoder_gru.candidate_mlp.weight.data = torch.from_numpy(param[26]).to(torch.float).t()
    net.basis_encoder_gru.candidate_mlp.bias.data = torch.from_numpy(param[27]).to(torch.float)

    net.basis_encoder_MLP[0].weight.data = torch.from_numpy(param[20]).to(torch.float).t()
    net.basis_encoder_MLP[0].bias.data = torch.from_numpy(param[21]).to(torch.float)
    net.basis_encoder_MLP[2].weight.data = torch.from_numpy(param[22]).to(torch.float).t()
    net.basis_encoder_MLP[2].bias.data = torch.from_numpy(param[23]).to(torch.float)

    net.basis_inference_MLP[0].weight.data = torch.from_numpy(param[16]).to(torch.float).t()
    net.basis_inference_MLP[0].bias.data = torch.from_numpy(param[17]).to(torch.float)
    net.basis_inference_MLP[2].weight.data = torch.from_numpy(param[18]).to(torch.float).t()
    net.basis_inference_MLP[2].bias.data = torch.from_numpy(param[19]).to(torch.float)

    net.prior_MLP[0].weight.data = torch.from_numpy(param[28]).to(torch.float).t()
    net.prior_MLP[0].bias.data= torch.from_numpy(param[29]).to(torch.float)
    net.prior_MLP[2].weight.data= torch.from_numpy(param[30]).to(torch.float).t()
    net.prior_MLP[2].bias.data= torch.from_numpy(param[31]).to(torch.float)

    net.decoder[0].weight.data = torch.from_numpy(param[32]).to(torch.float).t()
    net.decoder[0].bias.data = torch.from_numpy(param[33]).to(torch.float)
    net.decoder[2].weight.data = torch.from_numpy(param[34]).to(torch.float).t()
    net.decoder[2].bias.data = torch.from_numpy(param[35]).to(torch.float)

if __name__ == '__main__':
    # Parse command line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', type=str, help='Configuration file to load')
    args = parser.parse_args()

    # Load configuration file
    config = load(open(args.config, 'r'))
    obs_dim = config['obs_dim']
    latent_dim = config['latent_dim']
    control_dim = config['control_dim']
    del config['obs_dim']
    del config['latent_dim']
    del config['control_dim']

    config['subseq_len'] = 64
    config['predict_len'] = 32

    # Set random
    np.random.seed(config['seed'])
    torch.manual_seed(config['seed'])

    # Build feature extractor
    if config['feat_extractor_type'] == 'MLP':
        feat_size = config['feat_extractor_size']
        feat_act = ActivationList[config['feat_extractor_act']]
        feat_extractor = utils.construct_MLP(obs_dim, latent_dim, feat_size, feat_act)
    else:
        raise ValueError('Invalid type specified for feature extractor.')

    # Build decoder
    if config['decoder_type'] == 'MLP':
        decoder_size = config['decoder_size']
        decoder_act = ActivationList[config['decoder_act']]
        decoder = utils.construct_MLP(latent_dim, obs_dim, decoder_size, decoder_act)
    else:
        raise ValueError('Invalid type specified for decoder.')

    # Build the network
    rnn_size = config['rnn_size']
    transform_size = config['transform_size']
    inference_size = config['inference_size']
    prior_size = config['prior_size']
    del config['rnn_size']
    del config['transform_size']
    del config['inference_size']
    del config['prior_size']

    shift_x = torch.zeros(obs_dim)
    scale_x = torch.ones(obs_dim)
    net = VariationalKoopman(feat_extractor, decoder, latent_dim, control_dim, rnn_size, transform_size,
                             inference_size, prior_size, shift_x=shift_x, scale_x=scale_x, **config)

    # Import network weights
    net_weights = pickle.load(open('net.pkl', 'rb'))
    load_TF_weights(net, net_weights)

    # Weight on final state in the reconstruction loss
    if config['reconst_final_weight'] is not None:
        config['reconst_weight'] = torch.ones(config['subseq_len']-config['predict_len'])
        config['reconst_weight'][-1] = config['reconst_final_weight']
    else:
        config['reconst_weight'] = None

    # Build loss function
    loss_func = LossList[config['loss_func']]
    loss = loss_func(**config)

    # Get optimizer parameters
    optim = OptimList[config['optim']]
    optim_args = {'lr': config['lr']}

    # Get rollout policy
    if config['policy'] == 'RandomPolicy':
        policy = RandomPolicy(control_dim, config['policy_range'])
    else:
        raise ValueError['Invalid type specified for rollout policy.']

    # Load ReplayMemory dataset
    replay_buffer = pickle.load(open('replay_memory.pkl', 'rb'))
    x, x_val = torch.from_numpy(replay_buffer['x']), torch.from_numpy(replay_buffer['x_val'])
    u, u_val = torch.from_numpy(replay_buffer['u']), torch.from_numpy(replay_buffer['u_val'])
    x_test = torch.from_numpy(np.expand_dims(replay_buffer['x_test'], axis=0))
    u_test = torch.from_numpy(np.expand_dims(replay_buffer['u_test'], axis=0))
    shift_x, scale_x = torch.from_numpy(replay_buffer['shift_x']), torch.from_numpy(replay_buffer['scale_x'])
    shift_u, scale_u = torch.from_numpy(replay_buffer['shift_u']), torch.from_numpy(replay_buffer['scale_u'])

    dataset = TrajectoryDataset()
    dataset.states = torch.cat((x, x_val, x_test)).to(torch.float)
    dataset.obs = torch.cat((x, x_val, x_test)).to(torch.float)
    dataset.controls = torch.cat((u, u_val, u_test)).to(torch.float)
    dataset.scale_x = scale_x.to(torch.float)
    dataset.scale_u = scale_u.to(torch.float)
    dataset.shift_x = shift_x.to(torch.float)
    dataset.shift_u = shift_u.to(torch.float)
    dataset.train_idx = np.arange(0, x.shape[0])
    dataset.val_idx = np.arange(x.shape[0], x.shape[0]+x_val.shape[0])
    dataset.test_idx = [x.shape[0]+x_val.shape[0]]
    dataset.normalize = True
    config['dataset'] = dataset

    # Construct the experiment
    torch.set_printoptions(precision=10)
    exp_type = ExperimentList[config['env']]
    del config['env']
    exp = exp_type(net, loss, optimizer=optim, optim_args=optim_args, shuffle=False, **config)

    # Train model
    exp.train_model()


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /home/fsc/work/devel/flamingo/flamingo/plugins/menu/menu.py
# Compiled at: 2020-04-24 07:27:20
# Size of source mod 2**32: 2745 bytes
import logging, os
from flamingo.core.errors import MultipleObjectsReturned, ObjectDoesNotExist
from flamingo.core.data_model import Content, Q
logger = logging.getLogger('flamingo.plugins.Menu')

class Menu:
    THEME_PATHS = [
     os.path.join(os.path.dirname(__file__), 'theme')]

    def templating_engine_setup(self, context, templating_engine):

        def is_active(content, menu_item):
            return False

        def is_dict(v):
            return isinstance(v, dict)

        def is_list(v):
            return isinstance(v, list)

        templating_engine.env.globals['is_active'] = is_active
        templating_engine.env.globals['is_dict'] = is_dict
        templating_engine.env.globals['is_list'] = is_list

    def contents_parsed(self, context):

        def resolve_links(menu):
            for item in menu:
                name, url = item
                if isinstance(url, list):
                    resolve_links(url)
                else:
                    logger.debug('resolving %s', item[1])
                    try:
                        if isinstance(item[1], Content):
                            logger.debug('resolving skipped')
                            return
                        if isinstance(item[1], str):
                            lookup = Q(path=(item[1]))
                        else:
                            if not isinstance(item[1], Q):
                                lookup = Q(item[1])
                            else:
                                lookup = item[1]
                        item[1] = context.contents.get(lookup)
                        logger.debug('%s -> %s', lookup, item[1])
                    except ObjectDoesNotExist:
                        logger.error('no content with %s %s found', 'path' if isinstance(lookup, str) else 'lookup', lookup or repr(lookup))
                    except MultipleObjectsReturned:
                        logger.error('multiple contents found with %s %s found', 'path' if isinstance(lookup, str) else 'lookup', lookup or repr(lookup))

        if not hasattr(context.settings, 'MENU'):
            context.settings.MENU = {'main': []}
        else:
            if isinstance(context.settings.MENU, list):
                context.settings.MENU = {'main': context.settings.MENU}
            else:
                if 'main' not in context.settings.MENU:
                    context.settings.MENU['main'] = []
        for menu_name, menu in context.settings.MENU.items():
            resolve_links(menu)


def convert_dataframe_dtype(dataframe=None, Object_Category_threshold=10, Int_Category_threshold=0, Float_Category_threshold=0, verbose=False):
    """
    Method to convert the dataframe's dtype to appropiate dtyes.

    Convert the columns to Categorical data-type if the unique value count for that column is below threshold.
    Additionally also converts the datatypes to appropiate bit version based on number of unique values in it.

    --------------------
    PARAMETERS

    dataframe:
    Pandas DataFrame whose column's datatype needs to be converted

    Object_Category_threshold:
    Threshold (of count of unique values) of origional OBJECT columns. Only columns with less then threshold count
    would be converted into Categorical type.

    Int_Category_threshold:
    Threshold (of count of unique values) of origional INT columns. Only columns with less then threshold count
    would be converted into Categorical type.

    Float_Category_threshold:
    Threshold (of count of unique values) of origional FLOAT columns. Only columns with less then threshold count
    would be converted into Categorical type.

    --------------------
    RETURN:
    Converted DataFrame

    -------------------------------------------------------------------------------------------------------------------------------------------------

    Limitations:
    1.  When a int/float column is converted into a categorical variable, not all of the converted categorical columns represents the
        same dtype 'category'. Found that some are grouped into same 'category', while mamy other columns are showing as different 'category'

    2.  NaN value for numeric types are only expressed in 'float64' (by default), and can be cast to just 'float32'.
        NaN's cannot be cast to any integer or float16

    3.  IMPORTANT:
        Converting between/into floats requires extra attention.
        Float store data into two parts. Mantissa and Exponent.
        Although form scipy web-link the no of bits used in each section can be learnt, but somehow stil I am not sure about the exact precision
        level of each float*.
        When a INTEGER is stores in float datatype, it has diferece range of bits for precisely represent the number, but for floating point
        number the precision bits differ. More INVESTIGATION needed to find the exact range !!

        Due to this converting hight bits float to lower bit floats might cause authenticate value to be cast to 'inf'
        Obove situation is observed when a number which cannot be properly fit into lower bit represent, but is forced to cast into that.

    4.  SUGGESTION:
        I tries with converting some of the int/float columns with low unique counts to Categorical variables. The first motive was to save the
        space as much as possible and second motive was to represent the variable in more suitable type. This is because although some values are
        integer/float, they were actually representing a category measurement.

        Although the conversion is working prefectly, this leads to overhead later, especially during model building as most(almost all) models
        needs variables to be NUMERIC. So that would cause the datatype of such categorical features again to be changed to numeric.

        This has couple of issues.
        a. We need to find a way to know the data type in that categorical features. --OVERHEAD!!
        b. OR keep track what was the datatype of those features when it was converted into categorical. Might be in a list, dict. --OVERHEAD!!

        c. GROUPBY/ AGGREGATION problem: This is dependent on the case by case, as meaning of aggregation is diferent for categorical data. But found
        out that numeric groupby/aggregation operations do not be applied on categorical features, even though thay contain NUMERIC data.

        Also found that when we compared the memory usage of using categorical features -vs- lower bits numeric representation, there isn't huge
        difference. [Although this may be diferent if the values in that features is large, but this I belive is very rare.]

        So instead of converting numeric features to categorical, just converting them to lower bit numeric representation should be a better
        option to avoid all the later overhead.

        **Some additional steps we can perform is that after imputation, datatype convrsion can again be applied.
        This can help in reducing the float64 and float32 types to lower bit float16 representation.

        **Later, one possibility is that after imputation, a float feature which contains only integers, and will only contains integers, can be
        converted appropiately to integers if possible.

    -------------------------------------------------------------------------------------------------------------------------------------------------

    """
    #
    # Initial Memory usage in MB
    #
    gc.enable()
    #
    initial_memory_usage = dataframe.memory_usage(deep=True).sum() / (1024*1024)
    #
    for column in dataframe:
        #
        column_nuniue_count = dataframe[column].nunique()
        #
        # 1. Converting object types to categorical (if they have less then N number of values)
        if dataframe[column].dtype == 'object' and column_nuniue_count <= Object_Category_threshold:
            dataframe[column] = dataframe[column].astype('category')
        #
        #
        if dataframe[column].dtype == 'float64':
            #
            # Min and Max Value calculation only if column is float64.
            # The values range will be checked if they fit in a particualar type of not.
            column_max_value = dataframe[column].max()
            column_min_value = dataframe[column].min()
            null_value_flag = dataframe[column].isnull().any()
            #
            # Note: As Nan values can only correctly represented by float64 and float32, but not by float16,
            # so column contains Nan value should only be tried to cast to float32.
            if column_nuniue_count <= Float_Category_threshold:
                dataframe[column] = dataframe[column].astype('category')
            elif column_min_value >= -(2**11 -2) and column_max_value <= 2**11 - 2 and not null_value_flag:
            # float16 preserve the INTEGER number exactly till 2**12 -2 with 10 bits of mentisssa.
                dataframe[column] = dataframe[column].astype('float16')
            elif column_min_value >= -(2**24 -2) and column_max_value <= 2** 24 -2:
                dataframe[column] = dataframe[column].astype('float32')
        #
        #
        if dataframe[column].dtype == 'int64':
            #
            # Min and Max Value calculation only if column is int64.
            # The values range will be checked if they fit in a particualar type of not.
            column_max_value = dataframe[column].max()
            column_min_value = dataframe[column].min()
            #
            if column_nuniue_count <= Int_Category_threshold:
                dataframe[column] = dataframe[column].astype('category')
            elif  column_min_value >= -(2**7 - 1) and column_max_value <= (2**7 -1):
                dataframe[column] = dataframe[column].astype('int8')
            elif column_min_value >= -(2**15 - 1) and column_max_value <= (2**15 -1):
                dataframe[column] = dataframe[column].astype('int16')
            elif column_min_value >= -(2**31 - 1) and column_max_value <= (2**31 -1):
                dataframe[column] = dataframe[column].astype('int32')
        # Note: Any integer/float column having NaN value if by default cast to float while reading.
    #
    #
    final_memory_usage = dataframe.memory_usage(deep=True).sum() / (1024*1024)
    #
    if verbose:
        print('Reduction in Memory Percentage %f %%' % ((initial_memory_usage - final_memory_usage)*100 / initial_memory_usage))
        print('Reduction in Memory Usage: %f MB' % (initial_memory_usage - final_memory_usage))
    #
    gc.collect()
    #


from Tkinter import * #need to capitalize Tkinter to import

def do_nothing():
    print('Did nothing')

root = Tk() #sets up a window called root

menu_1 = Menu(root)
root.config(menu=menu_1) #menu is a funny word

sub_menu_1 = Menu(menu_1)
menu_1.add_cascade(label="File",menu=sub_menu_1)
sub_menu_1.add_command(label="New project",command=do_nothing)
sub_menu_1.add_command(label="Old project",command=do_nothing)
sub_menu_1.add_separator()
sub_menu_1.add_command(label="Blah",command=do_nothing)
sub_menu_1.add_separator()
sub_menu_1.add_command(label="Quit",command=root.quit)

menu_2 = Menu(menu_1)
menu_1.add_cascade(label="Edit",menu=menu_2)

root.mainloop() #runs the root


'''
Your function should take in a single parameter (a string `word`)
Your function should return a count of how many occurences of ***"th"*** occur within `word`. Case matters.
Your function must utilize recursion. It cannot contain any loops.
'''
def count_th(word, count=0):
    # need to define this here since we aren't passing it in the function
    substr = "th"

    #there can't be any more instances if there are less than 2 letters in the word
    if len(word) < len(substr):
        return count

    if word[0 : len(substr)] == substr:
        count += 1
        # if we get a match, loop again with the new word being all of the characters we haven't tested yet (word[len(substr) - 1:])
        return count_th(word[len(substr) - 1:], count)
    else:
        # else, we do the same thing except we don't increment the count var
        return count_th(word[len(substr) - 1:], count)

    return count

# just testing here
# print(count_th("abcthxyz"))


# -*- coding: utf-8 -*-
import scrapy

class CNNIntSpider(scrapy.Spider):
    name = 'CNN_int'
    allowed_domains = ['edition.cnn.com/world']
    start_urls = ['https://edition.cnn.com/world/']

    def parse(self, response):
        headers = response.xpath("//*[@id = 'world-zone-2']//h3[@class = 'cd__headline']/a/@href").extract()
        for header in headers:
            absolute_url = response.urljoin(header)
            yield scrapy.Request(absolute_url, callback=self.parse_news, dont_filter=True)

    def parse_news(self, response):

        title = response.css('h1::text').extract_first()

        posted_date = response.xpath("//p[@class = 'update-time']/text()").extract_first()

        story_body_introduction = response.xpath("//p[@class = 'zn-body__paragraph speakable']//text()").extract_first()

        img_url = response.xpath("//img[@class = 'media__image media__image--responsive']/@data-src-mini").extract_first()

        story_list = response.xpath('//*[@class="zn-body__paragraph speakable"]//text()').extract()
        information_src = "CNN"

        img_url = 'https:' + img_url if img_url else img_url

        full_story = ""
        story_list.pop(0)
        for story in story_list:
            full_story += story
            if full_story[-1] == '.':
                full_story += "\n\t"

        yield {
            'title': title,
            'img_url': img_url,
            'img_copyright': None,
            'story_body_introduction': story_body_introduction,
            'story_list': full_story,
            'posted_date': posted_date,
            'posted_time': None,
            'information_src': information_src
        }


import re, cPickle
from disco.util import err, data_err, msg

def netstr_reader(fd, content_len, fname):

    if content_len == None:
        err("Content-length must be defined for netstr_reader")
    def read_netstr(idx, data, tot):
        ldata = len(data)
        i = 0
        lenstr = ""
        if ldata - idx < 11:
            data = data[idx:] + fd.read(8192)
            ldata = len(data)
            idx = 0

        i = data.find(" ", idx, idx + 11)
        if i == -1:
            data_err("Corrupted input (%s). Could not "\
                   "parse a value length at %d bytes."\
                    % (fname, tot), fname)
        else:
            lenstr = data[idx:i + 1]
            idx = i + 1

        if ldata < i + 1:
            data_err("Truncated input (%s). "\
                "Expected %d bytes, got %d" %\
                (fname, content_len, tot), fname)

        try:
            llen = int(lenstr)
        except ValueError:
            data_err("Corrupted input (%s). Could not "\
                "parse a value length at %d bytes."\
                    % (fname, tot), fname)

        tot += len(lenstr)

        if ldata - idx < llen + 1:
            data = data[idx:] + fd.read(llen + 8193)
            ldata = len(data)
            idx = 0

        msg = data[idx:idx + llen]

        if idx + llen + 1 > ldata:
            data_err("Truncated input (%s). "\
                "Expected a value of %d bytes "\
                "(offset %u bytes)" %\
                (fname, llen + 1, tot), fname)

        tot += llen + 1
        idx += llen + 1
        return idx, data, tot, msg

    data = fd.read(8192)
    tot = idx = 0
    while tot < content_len:
        key = val = ""
        idx, data, tot, key = read_netstr(idx, data, tot)
        idx, data, tot, val = read_netstr(idx, data, tot)
        yield key, val

chain_reader = netstr_reader

def re_reader(item_re_str, fd, content_len, fname, output_tail = False, read_buffer_size=8192):
    item_re = re.compile(item_re_str)
    buf = ""
    tot = 0
    while True:
        if content_len:
            r = fd.read(min(read_buffer_size, content_len - tot))
        else:
            r = fd.read(read_buffer_size)
        tot += len(r)
        buf += r

        m = item_re.match(buf)
        while m:
            yield m.groups()
            buf = buf[m.end():]
            m = item_re.match(buf)

        if not len(r) or tot >= content_len:
            if content_len != None and tot < content_len:
                data_err("Truncated input (%s). "\
                     "Expected %d bytes, got %d" %\
                     (fname, content_len, tot), fname)
            if len(buf):
                if output_tail:
                    yield [buf]
                else:
                    msg("Couldn't match the last %d "\
                        "bytes in %s. Some bytes may be "\
                        "missing from input." %\
                        (len(buf), fname))
            break

def default_partition(key, nr_reduces, params):
    return hash(str(key)) % nr_reduces

def make_range_partition(min_val, max_val):
    r = max_val - min_val
    f = "lambda k, n, p: int(round(float(int(k) - %d) / %d * (n - 1)))" %\
        (min_val, r)
    return eval(f)

def nop_reduce(iter, out, params):
    for k, v in iter:
        out.add(k, v)

def map_line_reader(fd, sze, fname):
    for x in re_reader("(.*?)\n", fd, sze, fname, output_tail = True):
        yield x[0]

def netstr_writer(fd, key, value, params):
    skey = str(key)
    sval = str(value)
    fd.write("%d %s %d %s\n" % (len(skey), skey, len(sval), sval))

def object_writer(fd, key, value, params):
    skey = cPickle.dumps(key, cPickle.HIGHEST_PROTOCOL)
    sval = cPickle.dumps(value, cPickle.HIGHEST_PROTOCOL)
    fd.write("%d %s %d %s\n" % (len(skey), skey, len(sval), sval))

def object_reader(fd, sze, fname):
    for k, v in netstr_reader(fd, sze, fname):
        yield (cPickle.loads(k), cPickle.loads(v))

def map_input_stream(stream, size, url, params):
    m = re.match("(\w+)://", url)
    if m:
        scheme = m.group(1)
        try:
            mod = __import__("disco.schemes.scheme_%s" % scheme,
                    fromlist = ["scheme_%s" % scheme])
        except Exception:
            err("Unknown scheme %s in %s" % (scheme, url))
    else:
        from disco.schemes import scheme_file as mod
        url = "file://" + url
    mod.input_stream.func_globals.setdefault("Task", Task)
    return mod.input_stream(stream, size, url, params)

reduce_input_stream = map_input_stream

def map_output_stream(stream, partition, url, params):
    BUFFER_SIZE = int(1024**2)
    mpath, murl = Task.map_output(partition)
    if Task.num_partitions == 0:
        return disco.fileutils.AtomicFile(mpath, "w", BUFFER_SIZE), murl
    else:
        ppath, purl = Task.partition_output(partition)
        return disco.fileutils.PartitionFile(
            ppath, mpath, "w", BUFFER_SIZE), purl

def reduce_output_stream(stream, partition, url, params):
    BUFFER_SIZE = int(1024**2)
    path, url = Task.reduce_output()
    return disco.fileutils.AtomicFile(path, "w", BUFFER_SIZE), url


"""
    main parser function
    --sub_path: 분야별 폴더
    --dir_type: 문서 타입별 root directory. - TM(용어집)/AD(학술정보원문파일)/직접절대경로입력
"""

import argparse
from pathlib import Path

from parsers import xlsx_parser, pptx_parser, docx_mix_parser, docx_separate_parser, pdf_parser

from utils.regex_functions import *
from utils.common_functions import *

TM_ROOT = str(Path.home()) + "/Lexcode/AI 학습용 한영 말뭉치 구축 채널 - 인공지능 학습 DB 구축 채널 - 인공지능 학습 DB 구축 채널/1. 원본DB(렉스코드)/"
DOC_ROOT = str(Path.home()) + "/Lexcode/AI 학습용 한영 말뭉치 구축 채널 - 인공지능 학습 DB 구축 채널 - 인공지능 학습 DB 구축 채널/3.학술정보/"

# parameter variables
parser = argparse.ArgumentParser(
    description="Parsing .docx files to HTML raw text, then convert to excel file."
)

parser.add_argument(
    "-s",
    "--sub_path",
    type=str,
    required=True,
    help="<type: str> sub-directory for each category document files",
)

parser.add_argument(
    "-r",
    "--root_type",
    type=str,
    default="TM",
    help="<type: str> TM(default) or AD or type root_path manually. ex)/Users/twigfarm/LexcodeDrive/학술정보/",
)

args = parser.parse_args()

sub_path = args.sub_path
root_type = args.root_type

#  작업할 모든 파일 불러오기
def read_directory(root_type, sub_path):

    if root_type == "TM":
        root_dir = TM_ROOT
    elif root_type == "AD":
        root_dir = DOC_ROOT
    else:
        root_dir = root_type

    # read and categorize file type
    xlsx_file_list = get_filename_list(root_dir, sub_path, ".xlsx")
    xls_file_list = get_filename_list(root_dir, sub_path, ".xls")
    xlsx_files_list = xlsx_file_list + xls_file_list

    pptx_files_list = get_filename_list(root_dir, sub_path, ".pptx")
    docx_files_list = get_filename_list(root_dir, sub_path, ".docx")
    pdf_file_list = get_filename_list(root_dir, sub_path, ".pdf")

    # return excel, pptx, docx file list
    return (xlsx_files_list, pptx_files_list, docx_files_list, pdf_file_list)
    # return pptx_files_list

# RUN Parser
# 0:xlsx, 1:pptx, 2:docx
all_file_lists = read_directory(root_type, sub_path)

# pass each file type
xlsx_parser.xlsx_to_excel(all_file_lists[0], sub_path)
pptx_parser.pptx_to_excel(all_file_lists[1], sub_path)
docx_ko_files, docx_en_files, docx_mix_files, docx_non_lists = get_tm_doc_type(
    all_file_lists[2]
)
docx_separate_parser.docx_separate_to_excel(docx_ko_files, sub_path)
docx_mix_parser.docx_mix_to_excel(docx_mix_files, sub_path)
# pdf_parser.pdf_text_to_excel(all_file_lists[3], sub_path)


"""
This file defines functions and classes related to the registration and implementation of callbacks via python.
It currently contains:
- callbacks allowing V-Ray to support the AOV functionality of Maya Render Setup 2017 system.
"""

import maya.cmds as cmds
import maya.mel as mel
import maya.api.OpenMaya as OpenMaya
import maya.app.renderSetup.model.rendererCallbacks as rc
import maya.app.renderSetup.model.selector as selector
import maya.app.renderSetup.model.utils as utils
import maya.app.renderSetup.model.renderSetup as renderSetup

VRAY_RENDER_ELEMENT_NODE_TYPES = ["VRayRenderElement", "VRayRenderElementSet"]

def vrayGetExistingRenderElementsInScene():
    """ Returns a list of names of the existing render elements in the scene. """
    renderElementNames = []
    for reType in VRAY_RENDER_ELEMENT_NODE_TYPES:
        renderElementNames.extend(cmds.ls(exactType=reType))
    return renderElementNames

def isVRayRenderElementType(nodeType):
    """ Returns a boolean result depending on whether the given nodeType matches any V-Ray render element node type. """
    if not nodeType:
        return False
    return nodeType in VRAY_RENDER_ELEMENT_NODE_TYPES

class VRayRenderElementStrategy(selector.Strategy):
    """ The class provides a strategy for finding V-Ray render element nodes from a given input Selection. """

    @staticmethod
    def create(filterType, customs):
        # strategy of type VRayRenderElementStrategy will be returned only if the filter type is set to "Custom"
        # and at least one of the custom node types matches a V-Ray render element node type.
        if filterType == selector.Filters.kCustom and reduce(lambda out, nodeType: out or isVRayRenderElementType(nodeType), customs, False):
            return VRayRenderElementStrategy()
        return None

    def items(self, selection):
        return filter(lambda nodeName: isVRayRenderElementType(cmds.nodeType(nodeName)), selection.names())

    def onConnectionChanged(self, selectorNode, srcPlug, dstPlug, made):
        depNode = OpenMaya.MFnDependencyNode(dstPlug.node())
        if isVRayRenderElementType(depNode.typeName):
            selectorNode.selectionChanged()

    def isTraversingConnections(self):
        return True

class VRayAOVChildSelector(selector.SimpleSelector):
    """
    This selector is a node that is responsible for filtering V-Ray render element nodes
    in a child AOV collection in Render Setup. It derives from SimpleSelector and exists
    to override its strategy() method in order to fix improper behaviour ignoring nodes of type=VRayRenderElement.
    """

    kTypeId = OpenMaya.MTypeId(0x00126CC0)
    kTypeName = 'vrayAOVChildSelector'

    def strategy(self, dataBlock=None):
        if dataBlock or not self._strategy:
            filterType = self.getFilterType(dataBlock)
            customs = self.getTypeFilters() if filterType == selector.Filters.kCustom else []
            theStrategy = VRayRenderElementStrategy.create(filterType, customs)
            self._strategy = [] if theStrategy is None else theStrategy
        return self._strategy

class VRayAOVCallbacks(rc.AOVCallbacks):
    """
    Implements the AOV callbacks presented in Render Setup 2017 to provide support for export/import of V-Ray AOV data to an external file,
    opening the render settings window to "Render Elements" tab and organizing overrides of render elements into collections.
    """

    def encode(self):
        """ Encodes/exports the AOV information. """
        basicNodeExporter = rc.BasicNodeExporter()

        aovsDataJSON = {}
        renderElementNodes = []
        renderElementNames = vrayGetExistingRenderElementsInScene()

        for elemName in renderElementNames:
            basicNodeExporter.setNodes([elemName])
            # Passing the render element name to the exporter is enough in general but here we will set it ignore attributes
            # that are not created by this RE itself. This will be done removing the user defined from all the attributes.
            allPlugs = cmds.listAttr(elemName)
            userDefinedPlugs = cmds.listAttr(elemName, userDefined=True)
            for plug in userDefinedPlugs:
                allPlugs.remove(plug)
            plugsToIgnore = []
            for plug in allPlugs:
                fullAttrName = '{}.{}'.format(elemName, plug)
                plugsToIgnore.append(fullAttrName)
            basicNodeExporter.setPlugsToIgnore(plugsToIgnore)
            renderElementNodes.append( { elemName : basicNodeExporter.encode() })

        aovsDataJSON["renderElements"] = renderElementNodes
        return aovsDataJSON

    def decode(self, aovsData, decodeType):
        """
        Decodes/imports the AOV information.
        aovsData - The AOV data to decode
        decodeType - Overwrite, Merge
        Overwrite mode - specifies that the import should delete all existing AOV information before doing the import.
        Merge mode - overwrites any AOVs with the same name, imports AOVs with different names, and leaves alone any other pre-existing AOVs in the scene.
        """

        if decodeType == self.DECODE_TYPE_OVERWRITE:
            existingRenderElements = vrayGetExistingRenderElementsInScene()
            for elemName in existingRenderElements:
                mel.eval('vrayRemoveRenderElement "{}"'.format(elemName))

        basicNodeExporter = rc.BasicNodeExporter()
        renderElements = aovsData["renderElements"]
        for renderElement in renderElements:
            for elemName, elemData in renderElement.iteritems():
                fullAttrName = '{}.vrayClassType'.format(elemName)
                vrayClassType = elemData[fullAttrName]
                createdNodeName = mel.eval('vrayAddRenderElement "{}"'.format(vrayClassType))
                if createdNodeName != elemName:
                    cmds.rename(createdNodeName, elemName)
                basicNodeExporter.decode(elemData)

    def displayMenu(self):
        """ This function is called to display the Render Elements tab for V-Ray renderer in unifiedRenderGlobalsWindow. """

        mel.eval('unifiedRenderGlobalsWindow')
        mel.eval('setCurrentTabInRenderGlobalsWindow(\"Render Elements\")')
        mel.eval('fillSelectedTabForCurrentRenderer')

    def getAOVName(self, aovNode):
        """ From a given AOV node, returns the AOV name. The returned result is the collection name for the render element associated with aovNode."""

        nodeType = cmds.nodeType(aovNode)
        if nodeType in VRAY_RENDER_ELEMENT_NODE_TYPES:
            return '{}_col'.format(aovNode)
        return aovNode

    def getCollectionSelector(self, selectorName):
        """ This function is called to create the selector for the AOV collection and determines the node types allowed for it. """

        selectorName = cmds.createNode(selector.SimpleSelector.kTypeName, name=selectorName, skipSelect=True)
        selectorNode = utils.nameToUserNode(selectorName)
        selectorNode.setPattern("*")
        selectorNode.setFilterType(selector.Filters.kCustom)
        customFilterStr = ' '.join(VRAY_RENDER_ELEMENT_NODE_TYPES)
        selectorNode.setCustomFilterValue(customFilterStr)
        return selectorName

    def getChildCollectionSelector(self, selectorName, aovName):
        """
        This function is called to create the selector for the AOV child collection for the given render element.
        It uses a custom selector to filter the allowed types for the V-Ray render element associated with aovName.
        """
        selectorName = cmds.createNode(VRayAOVChildSelector.kTypeName, name=selectorName, skipSelect=True)
        selectorNode = utils.nameToUserNode(selectorName)
        aovNodeName = aovName[:-len('_col')] if aovName[-len('_col'):] == '_col' else aovName
        nodeType = cmds.nodeType(aovNodeName)
        selectorNode.setFilterType(selector.Filters.kCustom)
        selectorNode.setCustomFilterValue(nodeType)

        newSelection = OpenMaya.MSelectionList()
        newSelection.clear()
        newSelection.add(aovNodeName)
        selectorNode.staticSelection.setWithoutExistenceCheck(newSelection)
        return selectorName

    def getChildCollectionSelectorAOVNodeFromDict(self, d):
        """ This function returns the child selector AOV node name from the provided dictionary. """
        return d["selector"]["vrayAOVChildSelector"]["staticSelection"]

def vrayRegisterPythonCallbacks():
    """ The function registers the node VRayAOVChildSelector and the class VRayAOVCallbacks."""
    renderSetup.registerNode(VRayAOVChildSelector)
    # registering the set of AOV callbacks with the Render Setup system
    vrayAOVCallbacks = VRayAOVCallbacks()
    rc.registerCallbacks("vray", rc.CALLBACKS_TYPE_AOVS, vrayAOVCallbacks)

def vrayUnregisterPythonCallbacks():
    """ The function unregisters the node VRayAOVChildSelector and the AOV callbacks for Render Setup. """
    rc.unreregisterCallbacks("vray", rc.CALLBACKS_TYPE_AOVS)
    renderSetup.unregisterNode(VRayAOVChildSelector)


import re

from bs4 import BeautifulSoup

from mainapp.models import Group
from parserapp.parser.scrappers.rozklad.utils import RozkladRetryException

RE_NAME_CATHEDRA = re.compile(r'(.*) \((.*)\)')

async def get_teacher(session, url):
    async with session.post(url) as resp:
        html = await resp.text()

    def find_groups():
        for td in soup.find_all('td'):
            if td.find(class_='disLabel'):
                groups = list(td.children)[-1]
                groups = [i.group(0) for i in Group.RE_GROUP_CODE.finditer(groups)]
                yield from groups

    RozkladRetryException.check_html(html)
    soup = BeautifulSoup(html, features='lxml')
    header = soup.find(id='ctl00_MainContent_lblHeader')
    text = header.text.split('викладач: ')[1]

    full_name, cathedras_names = RE_NAME_CATHEDRA.search(text).groups()
    cathedras_names = cathedras_names.split(', ')

    return full_name, cathedras_names, find_groups()


import tkinter as tk
import game

class GameWindow:
    def __init__(self, players_color):
        self.players_color = players_color
        self.window = tk.Tk()
        self.window.geometry("1024x768")
        self.symbols = True
        self.this_game = game.Game(self.window, players_color, self)

        # creation labels for players and name of game
        self.l_monopoly = tk.Label(self.window, text="Monopoly", font=("Courier", 30))
        screen_width, screen_height = self.get_width_height()
        self.l_monopoly.place(x=0.4 * screen_width, y=0.3 * screen_height)
        self.l_current_player = tk.Label(
            self.window, text=f"Next: {self.this_game.players[self.this_game.current_player_index].name}", font=("Courier", 15))
        self.l_current_player.place(x=0.6 * screen_width, y=0.4 * screen_height)
        self.b_next_round = tk.Button(
            self.window, text="Next round", bg="MediumPurple4", fg="white", command=self.this_game.perform_round)
        self.b_next_round.place(x=0.6 * screen_width, y=0.6 * screen_height)
        self.l_players = []
        for index, i in enumerate(self.this_game.players):
            self.l_players.append(
                tk.Label(text=f"Player no {index + 1}: {i.name} {i.cash}", font=("Courier", 15), fg=i.name))
            self.l_players[index].place(x=0.5 * screen_width, y=(0.4 + index / 10) * screen_height)

        # buttons for situation when field doesnt have owner
        self.b_skip = tk.Button(self.window, text="Skip", bg="black",
                                fg="white", font=("Courier", 20))
        self.b_buy = tk.Button(self.window, text="Buy", bg="black",
                               fg="white", font=("Courier", 20))

        # button for situation where player on field is owner
        self.b_sell = tk.Button(self.window, text="Sell", bg="black",
                                fg="white", font=("Courier", 20))

        # button for paying rent
        self.b_pay = tk.Button(self.window, text="Pay", bg="black",
                               fg="white", font=("Courier", 20))

        # circles that represent players on map
        self.symbols = [None] * len(self.l_players)
        self.canvas = self.this_game.board.canvas
        self.on_resize()
        self.window.mainloop()

    def on_resize(self):
        screen_width, screen_height = self.get_width_height()
        self.l_monopoly.place(x=0.3 * screen_width, y=0.3 * screen_height)
        for index, i in enumerate(self.l_players):
            i.place(x=0.3 * screen_width, y=(0.4 + index / 10) * screen_height)
        self.l_current_player.place(x=0.6 * screen_width, y=0.4 * screen_height)
        self.b_next_round.place(x=0.6 * screen_width, y=0.6 * screen_height)
        self.draw_players()

    def get_width_height(self):
        self.window.update()
        return self.window.winfo_width(), self.window.winfo_height()

    def hide_menu(self):
        # self.l_monopoly.lower()
        self.l_current_player.lower()
        self.b_next_round.lower()
        # for i in self.l_players:
        # i.lower()

    def show_menu(self):
        self.this_game.current_player_index = (
            self.this_game.current_player_index + 1) % len(self.this_game.players)
        self.this_game.current_player = self.this_game.players[self.this_game.current_player_index]
        # self.l_monopoly.lift()
        self.l_current_player.lift()
        self.b_next_round.lift()
        for index, i in enumerate(self.l_players):
            j = self.this_game.players[index]
            i.config(text=f"Player no {index + 1}: {j.name} {j.cash}")
            # i.lift()
        self.l_current_player.config(
            text=f"Next: {self.this_game.players[self.this_game.current_player_index].name}")

    def show_offer(self, city, player):
        screen_width, screen_height = self.get_width_height()
        self.b_skip.place(x=0.5 * screen_width, y=0.5 * screen_height)
        self.b_buy.place(x=0.5 * screen_width, y=0.6 * screen_height)
        self.b_skip.lift()
        self.b_buy.lift()
        self.b_skip.config(command=self.buy_action)
        self.b_buy.config(command=lambda: [city.first_owner(
            player), self.buy_action()], text=f'Buy for {city.value}')

    def show_sell(self, city, player):
        screen_width, screen_height = self.get_width_height()
        self.b_skip.place(x=0.5 * screen_width, y=0.5 * screen_height)
        self.b_sell.place(x=0.5 * screen_width, y=0.6 * screen_height)
        self.b_skip.lift()
        self.b_sell.lift()
        self.b_skip.config(command=self.sell_action)
        self.b_sell.config(command=lambda: [city.sell_to_bank(
            player), self.sell_action()], text=f'Sell for {city.rent_value}')

    def pay_rent(self, city, player):
        screen_width, screen_height = self.get_width_height()
        self.b_pay.place(x=0.5 * screen_width, y=0.5 * screen_height)
        self.b_pay.lift()
        owner = self.this_game.find_owner(player.position)
        self.b_pay.config(command=lambda: [city.charge_rent(
            player, owner), self.pay_action()], text=f'Pay {city.rent_value}')

    def buy_action(self):
        self.b_skip.lower()
        self.b_buy.lower()
        self.show_menu()

    def sell_action(self):
        self.b_skip.lower()
        self.b_sell.lower()
        self.show_menu()

    def pay_action(self):
        self.b_pay.lower()
        self.show_menu()

    def remove_lplayer(self, index):
        delete_l = None
        for i, value in enumerate(self.l_players):
            if index == i:
                delete_l = value
        self.l_players.remove(delete_l)
        delete_l.destroy()

    def draw_players(self):
        # first we delete old symbols
        for i in self.symbols:
            self.canvas.delete(i)

        # then we draw new
        screen_width, screen_height = self.get_width_height()
        self.multipliers = [0.2, 0.4, 0.6, 0.8]  # array with relative positions inside of field
        for index, i in enumerate(self.this_game.players):
            position = i.position
            x = self.this_game.board.fields[position].x
            y = self.this_game.board.fields[position].y
            extra_width = self.multipliers[index] * 0.2 * screen_width
            extra_height = 0.1 * screen_height
            radius = 0.02 * screen_height
            att1 = x / 5 * screen_width + extra_width - radius
            att2 = y / 5 * screen_height + extra_height - radius
            att3 = x / 5 * screen_width + extra_width + radius
            att4 = y / 5 * screen_height + extra_height + radius
            self.symbols[index] = self.canvas.create_oval(att1, att2, att3, att4, fill=i.name)


import hashlib
import json

class NewCosmetics:
    def __init__(self, data):
        self.build = data.get('currentVersion')
        self.previousBuild = data.get('previousVersion')
        self.hash = hashlib.md5(self.build.encode()).hexdigest()
        self.items = [Cosmetic(i) for i in data.get('items')]

    def json(self):
        return json.dumps(self, default=lambda o: o.__dict__)

class Cosmetic:
    def __init__(self, data):
        self.id = data.get('id')
        self.name = data.get('name')
        self.description = data.get('description')

        self.type = CosmeticType(data)
        self.rarity = CosmeticRarity(data)
        self.series = CosmeticSeries(data) if data.get('series') else None
        self.set = CosmeticSet(data) if data.get('set') else None
        self.introduction = None
        self.images = Images(data.get('icons', {}))
        self.variants = data.get('variants')
        self.gameplayTags = [i for i in data.get('gameplayTags', [])] if data.get('gameplayTags') else []

class CosmeticType:
    def __init__(self, data):
        self.value = data.get('backendType')
        self.displayValue = data.get('shortDescription')

class CosmeticRarity:
    def __init__(self, data):
        self.backendValue = data.get('backendRarity')
        self.value = self.backendValue.split('::')[1].lower()
        self.displayValue = data.get('rarity')

class CosmeticSeries:
    def __init__(self, data):
        self.backendValue = data.get('backendRarity')
        self.value = data.get('series').get('name')
        self.image = data.get('icons').get('series')

class CosmeticSet:
    def __init__(self, data):
        self.value = data.get('set')
        self.text = data.get('setText')

class Images:
    def __init__(self, data):
        self.smallIcon = None
        self.icon = data.get('icon')
        self.featured = data.get('featured')

class Build:
    def __init__(self, data):
        self.build = data.get('version')
        self.mainKey = data.get('mainKey')
        dKeys = data.get('dynamicKeys')
        self.dynamicKeys = [DynamicKey(i, dKeys[i]) for i in dKeys.keys()]

    def json(self):
        return json.dumps(self, default=lambda o: o.__dict__)

class DynamicKey:
    def __init__(self, pakFilename, key):
        self.pakFilename = pakFilename
        self.pakGuid = None
        self.key = key


import numpy as np
import sys
sys.path.insert(0,"../../../5/aewiens/")
from uhf import UHF

class CEPA0:

    def __init__(self,mol,mints):

        uhf = UHF(mol,mints)
        self.Ehf = uhf.compute_energy()

        self.conv = uhf.conv
        self.maxiter = uhf.maxiter

        self.G = uhf.g
        self.C = uhf.C
        self.e = uhf.e
        self.nocc = uhf.nocc
        self.nvirt = uhf.norb - self.nocc

        self.t = np.zeros((self.nocc,self.nocc,self.nvirt,self.nvirt))
        self.Ec = 0.0

    def transform_integrals(self,g,C):
        return np.einsum("Pp,Pqrs->pqrs",C,
                    np.einsum("Qq,PQrs->Pqrs",C,
                        np.einsum("Rr,PQRs->PQrs",C,
                            np.einsum("Ss,PQRS->PQRs",C,g))))

    def compute_energy(self):
        """
        :Return: CCD correlation energy
        """
        nocc, nvirt, e, t = self.nocc, self.nvirt, self.e, self.t
        g = self.transform_integrals(self.G,self.C)

        o = slice(0,nocc)
        v = slice(nocc,nocc+nvirt)
        x = np.newaxis
        D = e[o,x,x,x] + e[x,o,x,x] - e[x,x,v,x] - e[x,x,x,v]
        Ep = 1.0/D

        for i in range(self.maxiter):
            ##  terms
            t1 = g[o,o,v,v]
            t2 = np.einsum("abcd,ijcd->ijab",g[v,v,v,v],t)
            t3 = np.einsum("klij,klab->ijab",g[o,o,o,o],t)
            t4 = np.einsum("akic,jkbc->ijab",g[v,o,o,v],t)

            ##  permute t4
            t4P = t4 - t4.transpose((1,0,2,3)) - t4.transpose((0,1,3,2)) + t4.transpose((1,0,3,2))

            ##  update t
            t = Ep*( t1 + 0.5*t2 + 0.5*t3 + t4P )

            ##  evaluate Ecorr
            Ec = 0.25 * np.sum(g[o,o,v,v] * t )
            dE = np.fabs(Ec - self.Ec)

            ##  save
            self.Ec = Ec
            self.t = t

            print       ("@CEPA {:3d} {:20.15f} {:20.15f}"  .format(i,Ec,dE) )

            if dE < self.conv: break

        return self.Ec


import jwt, time

class LFAuthToken(object):
    """Class to create tokens for auth with Livefyre services."""
    def __init__(self, user, domain, key, display_name=None, duration=86400):
        self.domain = domain
        self.duration = duration
        self.key = key
        self.user = user
        self.display_name = display_name

    def __str__(self):
        """Return the generated token string."""
        return self.token

    @property
    def token(self):
        """Create a signed token from inputs."""
        token = dict(expires=self.duration + time.time(),
                     user_id=self.user,
                     domain=self.domain)
        if self.display_name:
            token['display_name'] = self.display_name
        return jwt.encode(token, self.key)


from django.conf.urls import patterns, url, include
from rest_framework.urlpatterns import format_suffix_patterns

from artfence import views

urlpatterns = format_suffix_patterns(patterns('',
	url(r'^search/$', views.search.as_view(), name='search'),
	url(r'^register_user/$', views.register_user.as_view(), name='register_user'),
	url(r'^login/$', views.log_in.as_view(), name='log_in'),
	url(r'^post/$', views.post.as_view(), name='post'),
	url(r'^profile_search/$', views.profile_search.as_view(), name='profile_search'),
	url(r'^view_post/$', views.view_post.as_view(), name='view_post'),
	url(r'^delete_post/$', views.delete_post.as_view(), name='delete_post'),
	url(r'^upvote/$', views.upvote.as_view(), name='upvote'),
    url(r'^api-token-auth/', 'rest_framework.authtoken.views.obtain_auth_token')

))


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import logging
import requests

from lxml import html
from urllib2 import urlopen, URLError, urlparse
from publisher.models import PublisherModel

# Optional packages
try:
    import cssutils
    cssutils.log.setLevel(logging.FATAL)
except (ImportError, ):
    cssutils = None
try:
    from html5print import CSSBeautifier
except (ImportError, ):
    CSSBeautifier = None

from bs4 import BeautifulSoup

from django.db import models
from django import forms
from django.utils.translation import ugettext_lazy as _
from django.core.files.base import ContentFile
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey

try:
    from django.template import Template, Context
except ImportError:  # Django < 1.8
    from django.template.loader import Template, Context
from django.utils.safestring import mark_safe

from .settings import ARCHIVE_PAGE_ASYNCHRONOUSLY, METADATA_FORM, STORAGE, RELATIONS, FILE_TYPE_CHOICES
from .compatible import ModelFormField
from .archiveurl import relative_to_full_url
from .tasks import archive_page_components

logger = logging.getLogger(__name__)

class ArchivePageXpathForm(forms.Form):
    title = forms.CharField(
        required=True,
        initial='/html/head/title')
    content = forms.CharField(
        required=True,
        initial='/html/body')

class ArchivedPage(PublisherModel):
    """
    Part of an HTML page
    """
    url = models.CharField(
        _('URL'),
        max_length=255,
        db_index=True)
    original_url = models.CharField(
        _('original URL'),
        max_length=255)
    title = models.CharField(
        _('title'),
        blank=True,
        max_length=200)
    content = models.TextField(
        _('content'),
        blank=True)
    head = models.TextField(
        _('head'),
        blank=True)
    template_name = models.CharField(
        _('template name'),
        max_length=70,
        blank=True,
        help_text=_("""Example: 'vintage/contact_page.html'. If this
            isn't provided, the system will use 'vintage/wpf.html'."""))
    metadata = ModelFormField(form=METADATA_FORM)
    xpaths = ModelFormField(form=ArchivePageXpathForm)
    refetch = models.BooleanField(default=False)

    class Meta:
        verbose_name = _('archived page')
        verbose_name_plural = _('archived pages')
        ordering = ('url',)
        unique_together = (('url', 'publisher_is_draft'),
                           ('original_url', 'publisher_is_draft'))

    def __unicode__(self):
        return "%s -- %s" % (self.url, self.title)

    @models.permalink
    def get_absolute_url(self):
        return ('vintage_detail', (), {'url': self.url.lstrip('/')})

    def clone_relations(self, src_obj, dst_obj):
        '''
        Overrides PublisherModel.clone_relations() which is called at end of .publish().
            * publishes src_obj associated files.
            * clones ArchivedPageRelation for src_obj to dst_obj.
        '''
        # Clear all Files for dst_obj
        dst_obj.files.clear()
        # Publish each of src_obj's files, if not already published,
        # and link each published-file to dst_obj.
        for src_file in src_obj.files.all():
            if src_file.publisher_linked:
                dst_file = src_file.publisher_linked
            else:
                try:
                    dst_file = ArchivedFile.objects.get(original_url=src_file.original_url,
                                                        publisher_is_draft=False)
                except ArchivedFile.DoesNotExist:
                    src_file.publish()
                    dst_file = src_file.publisher_linked
            dst_file.archivedpages.add(dst_obj)
        # Clone all ArchivedPageRelations
        dst_obj.relations.all().delete()
        for src_relation in src_obj.relations.all():
            dst_relation = ArchivedPageRelation(
                archivedpage=dst_obj,
                content_type=src_relation.content_type,
                object_id=src_relation.object_id,
                alias=src_relation.alias,
            )
            dst_relation.save()

    def get_files(self, file_type=None):
        """Get associated files, optional file type"""
        if file_type:
            return self.files.filter(file_type=file_type)
        return self.files.all()

    def get_page(self):
        page = requests.get(self.original_url)
        tree = html.fromstring(page.content)
        xcontent = tree.xpath(self.xpaths['content'])[0]

        try:
            # Get the title
            self.title = html.tostring(tree.xpath(self.xpaths['title'])[0])
        except (Exception, ) as exc:
            logger.exception('Error parsing title: %s', unicode(exc))

        try:
            # Get the head
            self.head = html.tostring(tree.xpath('/html/head')[0])
        except (Exception, ):
            logger.exception('Error parsing head: %s', unicode(exc))

        # Get main content
        self.content = self.get_content(xcontent)

    def get_content(self, xhtml):
        return html.tostring(xhtml)

    @property
    def published_state(self):
        if not self.publisher_linked:
            return 'draft'
        if self.is_dirty:
            return 'unpublished changes'
        return 'published'

    def update_links(self, save=True):
        """
        Parse through the saved document and make sure the links to the existing
        site are archived.
        """
        soup = BeautifulSoup(self.content, "html.parser")
        links = soup.find_all('a')
        for tag in links:
            if not tag.has_key('href'):  # NOQA
                continue

            href = tag['href'].strip()
            if href.startswith('{'):
                continue
            if href.startswith('javascript'):
                continue
            if '.mp3' in href:
                # It's a Link pointing to an audio file.
                if not href.startswith('{'):
                    url = self.get_original_file(href, file_type='audio')
                    tag['href'] = url
            elif ('.png' in href or
                  '.jpg' in href or
                  '.jpeg' in href or
                  '.gif' in href):
                # It's a Link pointing to an image.
                if not href.startswith('{'):
                    url = self.get_original_file(href, file_type='image')
                    tag['href'] = url
            elif ('.doc' in href or
                  '.docx' in href or
                  '.pdf' in href or
                  '.xls' in href):
                # It's a link pointing to a document.
                if not href.startswith('{'):
                    url = self.get_original_file(href, file_type='document')
                    tag['href'] = url
            else:
                # It's a regular Link pointing to url for a page.
                url = relative_to_full_url(self.original_url, href)
                try:
                    ap = ArchivedPage.objects.get(original_url=url, publisher_is_draft=True)
                    url = "{%% url 'vintage_detail' url='%s' %%}" % ap.url.lstrip('/')
                except (ArchivedPage.DoesNotExist, ):
                    logger.warning('Unable to get archived page: %s', url)
                tag['href'] = url

        self.content = unicode(soup.prettify())
        if save:
            self.save()

    def strip_js(self, save=True):
        soup = BeautifulSoup(self.content, "html.parser")
        scripts = soup.find_all('script')
        [s.extract() for s in scripts]
        self.content = unicode(soup.prettify())
        if save:
            self.save()

    def update_audio_tags(self, save=True):
        """
        Parse through the saved document and make sure the images are archived.
        """
        soup = BeautifulSoup(self.content, "html.parser")
        audio_tags = soup.find_all('audio')
        for tag in audio_tags:
            src = tag['src'].strip()
            if not src.startswith('{'):
                url = self.get_original_file(src, file_type='audio')
                tag['src'] = url
        self.content = unicode(soup.prettify())
        if save:
            self.save()

    def update_images(self, save=True):
        """
        Parse through the saved document and make sure the images are archived.
        """
        soup = BeautifulSoup(self.content, "html.parser")
        images = soup.find_all('img')
        for tag in images:
            src = tag['src'].strip()
            if not src.startswith('{'):
                url = self.get_original_file(src, file_type='image')
                tag['src'] = url
        self.content = unicode(soup.prettify())
        if save:
            self.save()

    def update_css(self, save=True):
        """
        Parse the head of the document for css files.
        """
        soup = BeautifulSoup(self.head, "html.parser")
        # All the link head tags
        css = soup.find_all('link')
        for i, link in enumerate(css):
            # Make sure we are dealing with stylesheets
            if 'stylesheet' in link.get('rel', []):
                # The url of the stylesheet
                src = link['href'].strip()
                # if this has special markup, skip
                if not src.startswith('{'):
                    # Generate the local file as editable
                    url = self.get_original_file(
                        src, editable=True,
                        update_func=self.update_internal_css,
                        pretty=CSSBeautifier.beautify,
                        pretty_args=[4],  # Indent
                        order=i,
                        file_type='css')
                    link['href'] = url
        styles = soup.find_all('style')
        for style in styles:
            style.string = self.update_internal_css(self.original_url, style.string)
        self.head = unicode(soup.prettify())
        if save:
            self.save()

    def update_internal_css(self, original_path, content):
        """Given the contents of the css file, parse out any urls so we
        have them locally. Note: that if cssutils is not installed it
        will return the content as is.

        """
        def replace_url(url):
            """This is called with the cssutils.replaceUrls function"""
            parsed = urlparse.urlparse(original_path)
            # In the case where the path contains .., we make the url proper
            if url.startswith('..'):
                url = os.path.abspath(
                    os.path.join(
                        os.path.dirname(parsed.path), url))
                # Build the full url with domain so we can download it
                url = '{}://{}{}'.format(parsed.scheme, parsed.netloc, url)
            # Generate the local file for the url
            return self.get_original_file(url)

        # We can parse the css urls if cssutils is available
        if cssutils:
            sheet = cssutils.parseString(content)
            cssutils.replaceUrls(sheet, replace_url)
            return sheet.cssText
        return content

    def get_original_file(self, path, **kwargs):
        """
        Given a full or partial path, download and create the archivedfile.
        Return the url path instance
        """
        editable = kwargs.get('editable', False)
        update_func = kwargs.get('update_func', None)
        pretty = kwargs.get('pretty', None)
        pretty_args = kwargs.get('pretty_args', [])
        order = kwargs.get('order', 0)
        file_type = kwargs.get('file_type', '')

        if path.startswith('//'):
            path = 'http:%s' % path
        parsed = urlparse.urlparse(path.strip())
        path = relative_to_full_url(self.original_url, path.strip())

        # Get or create the archived file
        af, _ = ArchivedFile.objects.get_or_create(original_url=path,
                                                   publisher_is_draft=self.publisher_is_draft)
        af.order = order
        af.file_type = file_type
        af.save()

        try:
            try:
                file_content = urlopen(path).read()
            except (Exception, ) as exc:
                logger.exception('Error opening file: %s', unicode(exc))
                file_content = ''

            try:
                # Save the file contents
                af.content.save(
                    os.path.basename(parsed.path), ContentFile(file_content))
            except (Exception,) as exc:
                logger.exception('Error saving file: %s', unicode(exc))
                return ''

            # If we want the content of the file to be editable, we create an
            # addtional model record
            if editable:
                try:
                    source, _ = ArchivedSourceFile.objects.get_or_create(
                        archivedfile=af)

                    # Optinal function to handle the file contents
                    if update_func:
                        file_content = update_func(path, file_content)

                    # Optional function to prettyify the file contents
                    if pretty:
                        file_content = pretty(file_content, *pretty_args)

                    source.source = file_content
                    source.save()
                except (Exception, ) as exc:
                    logger.exception(
                        'Error creating editable srouce: %s', unicode(exc))
        except URLError:
            return path

        self.files.add(af)
        return '{{ STATIC_URL }}%s' % af.content.url

    def save(self, *args, **kwargs):
        if (not self.content) or self.refetch:
            if not self.id:
                super(ArchivedPage, self).save(*args, **kwargs)
            if ARCHIVE_PAGE_ASYNCHRONOUSLY:
                archive_page_components.delay(self)
            else:
                archive_page_components(self)

        super(ArchivedPage, self).save(*args, **kwargs)

    if RELATIONS:
        def get_related_content_type(self, content_type):
            """
            Get all related items of the specified content type
            """
            return self.relations.filter(content_type__name=content_type)

        def get_relation_type(self, relation_type):
            """
            Get all relations of the specified relation type
            """
            return self.relations.filter(alias__iexact=relation_type)

        def get_content_object(self, field):
            app_label, model = field[1].split('.')
            ctype = ContentType.objects.get(app_label=app_label, model=model)
            ar = self.get_related_content_type(ctype.name)
            if len(ar) > 0:
                return ar[0].content_object
            else:
                return None

def get_upload_path(instance, filename):
    """
    Return the path based on the primary_key of the related page
    """
    from urlparse import urlparse
    parsed = urlparse(instance.original_url)
    directory_name = os.path.normpath(
        os.path.join(
            'vintage',
            parsed.netloc,
            os.path.dirname(parsed.path).strip('/'))
    )
    new_filename = os.path.normpath(
        instance.content.storage.get_valid_name(
            os.path.basename(filename)))
    return os.path.join(directory_name, new_filename)

class ArchivedFile(PublisherModel):
    """
    A non-html file used in an Archived Page, such as a file
    """
    archivedpages = models.ManyToManyField(ArchivedPage, related_name='files')
    original_url = models.CharField(
        _('original URL'),
        max_length=255)
    content = models.FileField(
        max_length=255,
        upload_to=get_upload_path,
        storage=STORAGE())
    file_type = models.CharField(
        _('type'), choices=FILE_TYPE_CHOICES,
        blank=True,
        max_length=25)
    order = models.IntegerField(default=0)

    class Meta:
        ordering = ('order', )
        unique_together = (('original_url', 'publisher_is_draft'), )

    def __unicode__(self):
        return self.original_url

    def clone_relations(self, src_obj, dst_obj):
        '''
        When published(), runs this at the end to keep relations nsync:
            * Leaving this here as a note, be careful adding logic in here,
              initially tried cloning file-page relationships from here with strange
              results.  That work is being done in ArchivedPage.clone_relations().
        '''
        pass

    @property
    def published_state(self):
        if not self.publisher_linked:
            return 'draft'
        if self.is_dirty:
            return 'unpublished changes'
        return 'published'

class ArchivedSourceFile(models.Model):
    archivedfile = models.OneToOneField(ArchivedFile)
    source = models.TextField(_('source'))

    def save(self, *args, **kwargs):
        # Only update the source file if the `source` field has data
        from django.core.files.base import ContentFile

        if self.source:
            path = self.archivedfile.original_url
            if path.startswith('//'):
                path = 'http:%s' % path

            # Render the source content
            source = mark_safe(Template(self.source).render(Context({})))

            # Fetch the storage and name of the content field
            storage = self.archivedfile.content.storage
            file_name = self.archivedfile.content.file.name
            try:
                # generate the archived filed using the source
                storage.save(file_name, ContentFile(source))
            except (Exception, ) as exc:
                logger.exception('Error writing source file: %s', unicode(exc))

        super(ArchivedSourceFile, self).save(*args, **kwargs)

if RELATIONS:
    ARCHIVEDPAGE_RELATION_LIMITS = reduce(lambda x, y: x | y, RELATIONS)
else:
    ARCHIVEDPAGE_RELATION_LIMITS = []

class ArchivedPageRelationManager(models.Manager):
    def get_content_type(self, content_type):
        """
        Get all the items of the given content type related to this item.
        """
        qs = self.get_query_set()
        return qs.filter(content_type__name=content_type)

    def get_relation_type(self, relation_type):
        """
        Get all the items of the given relationship type related to this item.
        """
        qs = self.get_query_set()
        return qs.filter(relation_type=relation_type)

class ArchivedPageRelation(models.Model):
    """Related item"""
    archivedpage = models.ForeignKey(
        ArchivedPage, related_name='relations')
    content_type = models.ForeignKey(
        ContentType, limit_choices_to=ARCHIVEDPAGE_RELATION_LIMITS)
    object_id = models.PositiveIntegerField(
        blank=False, null=False)
    content_object = GenericForeignKey(
        ct_field='content_type', fk_field='object_id')
    alias = models.CharField(
        verbose_name=_("Alias"), max_length=200, blank=True, null=True,
        help_text=_("A generic text field to tag a relation, "
                    "like 'leadphoto'."))

    objects = ArchivedPageRelationManager()

    def __unicode__(self):
        return u"ArchivedPageRelation"


# import necessary packages
import imutils
import cv2
import numpy as np

def objCenter(rects, frameCenter):
	# check to see if a object was found
	if len(rects) > 0:
		# find the largest contour in the mask, then use
		# it to compute the minimum enclosing circle and
		# centroid
		c = max(rects, key=cv2.contourArea)
		((x, y), radius) = cv2.minEnclosingCircle(c)
		M = cv2.moments(c)
		center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))

		objectX = center[0]
		objectY = center[1]

		# return the center (x, y)-coordinates of the face
		return ((objectX, objectY), radius)

	# otherwise no faces were found, so return the center of the
	# frame
	return (frameCenter, None)


"""GAOGDLV URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.8/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import include, url
from django.contrib import admin

from GAOGDS import views as base_views

urlpatterns = [
    url(r'^admin/', include(admin.site.urls)),
    url(r'^$', base_views.index,name='index'),
    url(r'^login/$', base_views.account_login, name='login'),
    url(r'^logout/$', base_views.account_logout, name='logout'),
    url(r'^service/(?P<type>(.*)$)?', base_views.service, name='service'),
    url(r'^file/(?P<type>(.*)$)?', base_views.file, name='file'),
    url(r'^group', base_views.group, name='group'),
]


import brian2 as b2
from neurodynex.hodgkin_huxley import HH
#b2.defaultclock.dt = 1 * b2.ms
#import brian2 as b2
second = 1000*b2.units.ms
b2.A = 1000000000000*b2.pA
b2.units.V = 1000.0*b2.units.mV
# Hodgkin Huxley parameters
'''
# Parameters
Cm = 1*b2.ufarad*cm**-2 * area
gl = 5e-5*siemens*cm**-2 * area
El = -65*mV
EK = -90*mV
ENa = 50*mV
g_na = 100*msiemens*cm**-2 * area
g_kd = 30*msiemens*cm**-2 * area
VT = -63*mV
'''
#b2.A = 1000000000000*b2.pA
from neurodynex.tools import plot_tools, input_factory
import io
import math
import pdb
from numba import jit
import numpy as np
from .base import *
import quantities as pq

from quantities import mV as qmV
from quantities import ms as qms
from quantities import V as qV
#pq.PREFERRED = [pq.mV, pq.pA, pq.UnitQuantity('femtocoulomb', 1e-15*pq.C, 'fC')]

SLOW_ZOOM = False
#, ms, s, us, ns, V
import matplotlib as mpl
from neuronunit.capabilities import spike_functions as sf
mpl.use('Agg')
import matplotlib.pyplot as plt
from elephant.spike_train_generation import threshold_detection

getting_started = False
try:
    import asciiplotlib as apl
    fig = apl.figure()
    fig.plot([1,0], [0,1])
    ascii_plot = True
    import gc

except:
    ascii_plot = False
import numpy
try:
    brian2.clear_cache('cython')
except:
    pass

from neuronunit.capabilities.spike_functions import get_spike_waveforms

def simulate_HH_neuron_local(input_current=None,
                            st=None,
                            El=None,\
                            EK=None,
                            ENa=None,
                            gl=None,\
                            gK=None,
                            gNa=None,
                            C=None,
                            Vr=None):
    # code lifted from:
    # /usr/local/lib/python3.5/dist-packages/neurodynex/hodgkin_huxley
    #input_current = I_stim #= #stim, simulation_time=st)
    """A Hodgkin-Huxley neuron implemented in Brian2.

    Args:
        input_current (TimedArray): Input current injected into the HH neuron
        st (float): Simulation time [seconds]

    Returns:
        StateMonitor: Brian2 StateMonitor with recorded fields
        ["vm", "I_e", "m", "n", "h"]

    https://brian2.readthedocs.io/en/stable/examples/IF_curve_Hodgkin_Huxley.html
    area = 20000*umetre**2
    Cm = 1*ufarad*cm**-2 * area
    gl = 5e-5*siemens*cm**-2 * area
    El = -65*mV
    EK = -90*mV
    ENa = 50*mV
    g_na = 100*msiemens*cm**-2 * area
    g_kd = 30*msiemens*cm**-2 * area
    VT = -63*mV
    """
    area = 20000*b2.umetre**2
    Cm = float(C) *b2.ufarad*b2.cm**-2 * area
    VT = -63*b2.mV
    assert float(El)<0.0
    # The model
    eqs = ('''
    dvm/dt = (gl*(El-vm) - g_na*(m*m*m)*h*(vm-ENa) - g_kd*(n*n*n*n)*(vm-EK) + input_current)/Cm : volt
    dm/dt = 0.32*(mV**-1)*4*mV/exprel((13.*mV-vm+VT)/(4*mV))/ms*(1-m)-0.28*(mV**-1)*5*mV/exprel((vm-VT-40.*mV)/(5*mV))/ms*m : 1
    dn/dt = 0.032*(mV**-1)*5*mV/exprel((15.*mV-vm+VT)/(5*mV))/ms*(1.-n)-.5*exp((10.*mV-vm+VT)/(40.*mV))/ms*n : 1
    dh/dt = 0.128*exp((17.*mV-vm+VT)/(18.*mV))/ms*(1.-h)-4./(1+exp((40.*mV-vm+VT)/(5.*mV)))/ms*h : 1
    input_current : amp
    ''')
    # Threshold and refractoriness are only used for spike counting
    neuron = b2.NeuronGroup(1, eqs,
                        threshold='v > -40*mV',
                        refractory='v > -40*mV',
                        method='exponential_euler')
    neuron.vm = El#*b2.units.mV

    #neuron.v = El
    #neuron = b2.NeuronGroup(1, eqs, method="exponential_euler")
    # parameter initialization
    #neuron.m = 0.05
    #neuron.h = 0.60
    #neuron.n = 0.32

    #spike_monitor = b2.SpikeMonitor(neuron)
    # tracking parameters
    st_mon = b2.StateMonitor(neuron, ["vm"], record=True)

    # running the simulation
    neuron = b2.Network(neuron)
    neuron.add(st_mon)
    # dur0 = 0.1*second
    neuron.I = '0.0*nA'

    dur0 = int(input_current['delay'])*second
    neuron.run(dur0)
    amp = input_current['amp']

    neuron.I = str(amp)+str('nA')
    dur1 = int(input_current['delay']+input_current['duration'])*second
    neuron.run(dur1)
    dur2 = 0.2*second
    neuron.I = '0.0*nA'
    neuron.run(dur2)
    import pdb
    pdb.set_trace()
    vm_b = neuron.vm
    vm_b = [ float(i) for i in vm_b ]
    vm_b = AnalogSignal(vm_b,units = pq.V,sampling_period = float(0.001) * pq.s)
    self.vM = vm_b
    """
    eqs =
    I_e = input_current(t,i) : amp
    membrane_Im = I_e + gNa*m**3*h*(ENa-vm) + \
        gl*(El-vm) + gK*n**4*(EK-vm) : amp
    alphah = .07*exp(-.05*vm/mV)/ms    : Hz
    alpham = .1*(25*mV-vm)/(exp(2.5-.1*vm/mV)-1)/mV/ms : Hz
    alphan = .01*(10*mV-vm)/(exp(1-.1*vm/mV)-1)/mV/ms : Hz
    betah = 1./(1+exp(3.-.1*vm/mV))/ms : Hz
    betam = 4*exp(-.0556*vm/mV)/ms : Hz
    betan = .125*exp(-.0125*vm/mV)/ms : Hz
    dh/dt = alphah*(1-h)-betah*h : 1
    dm/dt = alpham*(1-m)-betam*m : 1
    dn/dt = alphan*(1-n)-betan*n : 1
    dvm/dt = membrane_Im/C : volt
    """

    return st_mon,selfvM,vm

    """
    state_dic = st_mon.get_states()
    vm = state_dic['vm']
    v_nan = []
    for v in vm:
        v = v*1000.0
        if np.isnan(v):
           v_nan.append(-65.0*b2.units.mV)
        else:
           v_nan.append(v)
    vM = AnalogSignal(v_nan,units = pq.mV,sampling_period = float(0.001) * pq.s)
    """
    #vM = AnalogSignal(v_nan,units = pq.mV,sampling_period = 1*pq.ms)#b2.defaultclock.dt*pq.s)
    '''
    try:
     	vM.rescale_prefered()
    except:
		import pdb
		pdb.set_trace()
    '''

getting_started = False
class BHHBackend(Backend):

    name = 'BHH'

    def init_backend(self, attrs=None, cell_name='thembi',
                     current_src_name='spanner', DTC=None,
                     debug = False):

        super(BHHBackend,self).init_backend()

        self.model._backend.use_memory_cache = False
        self.current_src_name = current_src_name
        self.cell_name = cell_name
        self.vM = None
        self.attrs = attrs
        self.debug = debug
        self.temp_attrs = None
        self.n_spikes = None
        self.verbose = False

        if type(attrs) is not type(None):
            self.set_attrs(attrs)
            self.sim_attrs = attrs

        if type(DTC) is not type(None):
            if type(DTC.attrs) is not type(None):
                self.set_attrs(DTC.attrs)
            if hasattr(DTC,'current_src_name'):
                self._current_src_name = DTC.current_src_name
            if hasattr(DTC,'cell_name'):
                self.cell_name = DTC.cell_name

    def get_spike_count(self):
        #if np.max(self.vM)>20.0*np.mean(self.vM):
        #thresh = threshold_detection(self.vM,np.max(self.vM)-0.10*np.max(self.vM))
        thresh = threshold_detection(self.vM,0.0*pq.mV)

        #else:
        #    thresh = []
        return len(thresh)

    def set_stop_time(self, stop_time = 650*pq.ms):
        """Sets the simulation duration
        stopTimeMs: duration in milliseconds
        """
        self.tstop = float(stop_time.rescale(pq.ms))

    def get_membrane_potential(self):
        """Must return a neo.core.AnalogSignal.
        And must destroy the hoc vectors that comprise it.
        """

        return self.vM

    def set_attrs(self,attrs):

        self.HH = None
        self.HH = HH

        if len(attrs):
            self.El = attrs['El'] * b2.units.mV
            self.EK = attrs['EK'] * b2.units.mV
            self.ENa = attrs['ENa'] * b2.units.mV
            self.gl =  attrs['gl'] * b2.units.msiemens
            self.gK = attrs['gK'] * b2.units.msiemens
            self.gNa = attrs['gNa'] * b2.units.msiemens
            self.C = attrs['C'] * b2.units.ufarad
            self.Vr = attrs['Vr']

            self.model.attrs.update(attrs)
        if attrs is None:
            #b2.defaultclock.dt = 1 * b2.ms

            self.HH =HH

    def inject_square_current(self, current):#, section = None, debug=False):
        """Inputs: current : a dictionary with exactly three items, whose keys are: 'amplitude', 'delay', 'duration'
        Example: current = {'amplitude':float*pq.pA, 'delay':float*pq.ms, 'duration':float*pq.ms}}
        where \'pq\' is a physical unit representation, implemented by casting float values to the quanitities \'type\'.
        Description: A parameterized means of applying current injection into defined
        Currently only single section neuronal models are supported, the neurite section is understood to be simply the soma.

        """
        #b2.defaultclock.dt = 1 * b2.ms
        self.state_monitor = None
        #self.spike_monitor = None
        self.HH = None
        self.HH = HH
        attrs = copy.copy(self.model.attrs)
        if self.model.attrs is None or not len(self.model.attrs):
            self.HH = HH
        else:
            self.set_attrs(attrs)
        if 'injected_square_current' in current.keys():
            c = current['injected_square_current']
        else:
            c = current

        duration = int(c['duration'])#/10.0)#/dt#/dt.rescale('ms')
        delay = int(c['delay'])#/10.0)#/dt#.rescale('ms')
        pre_current = int(duration)+100
        amp = c['amplitude']#.rescale('uA')

        """
        #amplitude = amp.simplified#/1000000.0
        params = {'delay':delay, 'duration':duration, 'amp':amp,'':}
        getting_started = False
        if getting_started == False:
            stim = input_factory.get_step_current(delay, duration, b2.ms, amp * b2.pA)
            st = (duration+delay+100)* b2.ms
        else:
            stim = input_factory.get_step_current(10, 7, b2.ms, 45.0 * b2.nA)

            st = 70 * b2.ms
	"""
        if self.model.attrs is None or not len(self.model.attrs):

            self.HH = HH
            self.state_monitor,self.vM = self.HH.simulate_HH_neuron(I_stim = {'delay':delay,'duration':duration,'amp':amp}, simulation_time=st)

        else:
            if self.verbose:
                print(attrs)
            self.set_attrs(attrs)

            (self.state_monitor,self.vM,vm) = simulate_HH_neuron_local(
            El = attrs['El'] * b2.units.V,
            EK = attrs['EK'] * b2.units.V,
            ENa = attrs['ENa'] * b2.units.V,
            gl =  attrs['gl'] * b2.units.msiemens,
            gK = attrs['gK'] * b2.units.msiemens,
            gNa = attrs['gNa'] * b2.units.msiemens,
            C = attrs['C'] * b2.units.ufarad,
            Vr = attrs['Vr'],
            input_current = {'delay':delay,'duration':duration,'amp':amp}
            )
            #params = params)

        #self.state_monitor.clock.dt = 1 *b2.ms
        self.dt = self.state_monitor.clock.dt

        self.attrs = attrs

        if ascii_plot:
            SLOW_ZOOM = False
            if SLOW_ZOOM and self.get_spike_count()>=1 :

                vm = get_spike_waveforms(self.vM)
            else:
                vm = self.vM
            t = [float(f) for f in vm.times]
            v = [float(f) for f in vm.magnitude]
            fig = apl.figure()
            fig.plot(t, v, label=str('brian HH: ')+str(vm.units)+str(current['amplitude']), width=100, height=20)

            fig.show()
            gc.collect()
            fig = None

        fig  = None
        return self.vM

    def _backend_run(self):
        results = None
        results = {}
        results['vm'] = self.vM
        results['t'] = self.vM.times
        results['run_number'] = results.get('run_number',0) + 1
        return results


#calss header
class _UNSHAKEABLE():
	def __init__(self,):
		self.name = "UNSHAKEABLE"
		self.definitions = [u"If someone's trust or belief is unshakeable, it is firm and cannot be made weaker or destroyed: "]

		self.parents = []
		self.childen = []
		self.properties = []
		self.jsondata = {}

		self.specie = 'adjectives'

	def run(self, obj1, obj2):
		self.jsondata[obj2] = {}
		self.jsondata[obj2]['properties'] = self.name.lower()
		return self.jsondata