from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import csv
import json
import os
import time

import numpy as np
import tensorflow as tf

from bert import modeling
from bert import optimization
from bert import tokenization

MIN_FLOAT = -1e30

flags = tf.flags
FLAGS = flags.FLAGS

flags.DEFINE_string("bert_config_file", None, "The config json file corresponding to the pre-trained BERT model.")
flags.DEFINE_string("vocab_file", None, "The vocabulary file that the BERT model was trained on.")
flags.DEFINE_string("data_dir", None, "The input data dir. Should contain the .json files for the task.")
flags.DEFINE_string("task_name", None, "The name of the task to train.")
flags.DEFINE_string("output_dir", None, "The output directory where the model checkpoints will be written.")
flags.DEFINE_string("export_dir", None, "The export directory where the saved model will be written.")

flags.DEFINE_string("init_checkpoint", None, "Initial checkpoint (usually from a pre-trained BERT model).")
flags.DEFINE_bool("do_lower_case", True, "Whether to lower case the input text. True for uncased models and False for cased models.")

flags.DEFINE_integer("random_seed", 100, "Random seed for weight initialzation.")
flags.DEFINE_string("predict_tag", None, "Predict tag for predict result tracking.")

flags.DEFINE_integer(
    "max_seq_length", 128,
    "The maximum total input sequence length after WordPiece tokenization. "
    "Sequences longer than this will be truncated, and sequences shorter than this will be padded.")

flags.DEFINE_bool("do_train", False, "Whether to run training.")
flags.DEFINE_bool("do_eval", False, "Whether to run evaluation.")
flags.DEFINE_bool("do_predict", False, "Whether to run prediction.")
flags.DEFINE_bool("do_export", False, "Whether to run exporting.")

flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.")
flags.DEFINE_integer("eval_batch_size", 8, "Total batch size for eval.")
flags.DEFINE_integer("predict_batch_size", 8, "Total batch size for predict.")

flags.DEFINE_float("learning_rate", 5e-5, "The initial learning rate for Adam.")
flags.DEFINE_float("num_train_epochs", 3.0, "Total number of training epochs to perform.")
flags.DEFINE_float("warmup_proportion", 0.1, "Proportion of training to perform linear learning rate warmup for.")

flags.DEFINE_integer("save_checkpoints_steps", 1000, "How often to save the model checkpoint.")
flags.DEFINE_integer("iterations_per_loop", 1000, "How many steps to make in each estimator call.")

flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.")
flags.DEFINE_integer("num_tpu_cores", 8,"Only used if `use_tpu` is True. Total number of TPU cores to use.")
flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.")

flags.DEFINE_string(
    "tpu_name", None,
    "The Cloud TPU to use for training. This should be either the name "
    "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 url.")

flags.DEFINE_string(
    "tpu_zone", None,
    "[Optional] GCE zone where the Cloud TPU is located in. If not "
    "specified, we will attempt to automatically detect the GCE project from metadata.")

flags.DEFINE_string(
    "gcp_project", None,
    "[Optional] Project name for the Cloud TPU-enabled project. If not "
    "specified, we will attempt to automatically detect the GCE project from metadata.")

class InputExample(object):
    """A single training/test example for simple sequence classification."""
    def __init__(self,
                 guid,
                 text,
                 sent_label=None):
        """Constructs a InputExample.

        Args:
          guid: Unique id for the example.
          text: string. The untokenized text of the first sequence. For single
            sequence tasks, only this sequence must be specified.
          sent_label: (Optional) string. The sentence label of the example. This should be
            specified for train and dev examples, but not for test examples.
        """
        self.guid = guid
        self.text = text
        self.sent_label = sent_label

class PaddingInputExample(object):
    """Fake example so the num input examples is a multiple of the batch size.

    When running eval/predict on the TPU, we need to pad the number of examples
    to be a multiple of the batch size, because the TPU requires a fixed batch
    size. The alternative is to drop the last batch, which is bad because it means
    the entire output data won't be generated.

    We use this class instead of `None` because treating `None` as padding
    battches could cause silent errors.
    """

class InputFeatures(object):
    """A single set of features of data."""
    def __init__(self,
                 input_ids,
                 input_masks,
                 segment_ids,
                 sent_label_id):
        self.input_ids = input_ids
        self.input_masks = input_masks
        self.segment_ids = segment_ids
        self.sent_label_id = sent_label_id

class ClassificationProcessor(object):
    """Processor for the classification data set."""
    def __init__(self,
                 data_dir,
                 task_name):
        self.data_dir = data_dir
        self.task_name = task_name

    def get_train_examples(self):
        """Gets a collection of `InputExample`s for the train set."""
        data_path = os.path.join(self.data_dir, "train-{0}".format(self.task_name), "train-{0}.json".format(self.task_name))
        data_list = self._read_json(data_path)
        example_list = self._get_example(data_list)
        return example_list

    def get_dev_examples(self):
        """Gets a collection of `InputExample`s for the dev set."""
        data_path = os.path.join(self.data_dir, "dev-{0}".format(self.task_name), "dev-{0}.json".format(self.task_name))
        data_list = self._read_json(data_path)
        example_list = self._get_example(data_list)
        return example_list

    def get_test_examples(self):
        """Gets a collection of `InputExample`s for the test set."""
        data_path = os.path.join(self.data_dir, "test-{0}".format(self.task_name), "test-{0}.json".format(self.task_name))
        data_list = self._read_json(data_path)
        example_list = self._get_example(data_list)
        return example_list

    def get_sent_labels(self):
        """Gets the list of sentence labels for this data set."""
        data_path = os.path.join(self.data_dir, "resource", "sent_label.vocab")
        sent_labels = self._read_text(data_path)
        return sent_labels

    def _read_text(self,
                   data_path):
        if os.path.exists(data_path):
            with open(data_path, "rb") as file:
                data_list = []
                for line in file:
                    data_list.append(line.decode("utf-8").strip())

                return data_list
        else:
            raise FileNotFoundError("data path not found")

    def _read_json(self,
                   data_path):
        if os.path.exists(data_path):
            with open(data_path, "r") as file:
                data_list = json.load(file)
                return data_list
        else:
            raise FileNotFoundError("data path not found")

    def _get_example(self,
                     data_list):
        example_list = []
        for data in data_list:
            guid = data["id"]
            text = tokenization.convert_to_unicode(data["text"])
            sent_label = tokenization.convert_to_unicode(data["sent_label"])
            example = InputExample(guid=guid, text=text, sent_label=sent_label)
            example_list.append(example)

        return example_list

def convert_single_example(ex_index,
                           example,
                           sent_label_list,
                           max_seq_length,
                           tokenizer):
    """Converts a single `InputExample` into a single `InputFeatures`."""
    if isinstance(example, PaddingInputExample):
        return InputFeatures(
            input_ids=[0] * max_seq_length,
            input_masks=[0] * max_seq_length,
            segment_ids=[0] * max_seq_length,
            sent_label_id=0)

    sent_label_map = {}
    for (i, sent_label) in enumerate(sent_label_list):
        sent_label_map[sent_label] = i

    tokens = tokenizer.tokenize(example.text)

    if len(tokens) > max_seq_length - 2:
        tokens = tokens[0:(max_seq_length - 2)]

    # The convention in BERT is:
    # (a) For sequence pairs:
    #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
    #  type_ids: 0     0  0    0    0     0       0 0     1  1  1  1   1 1
    # (b) For single sequences:
    #  tokens:   [CLS] the dog is hairy . [SEP]
    #  type_ids: 0     0   0   0  0     0 0
    #
    # Where "type_ids" are used to indicate whether this is the first
    # sequence or the second sequence. The embedding vectors for `type=0` and
    # `type=1` were learned during pre-training and are added to the wordpiece
    # embedding vector (and position vector). This is not *strictly* necessary
    # since the [SEP] token unambiguously separates the sequences, but it makes
    # it easier for the model to learn the concept of sequences.
    #
    # For classification tasks, the first vector (corresponding to [CLS]) is
    # used as the "sentence vector". Note that this only makes sense because
    # the entire model is fine-tuned.

    input_tokens = []
    segment_ids = []
    sent_label_id = sent_label_map[example.sent_label]

    input_tokens.append("[CLS]")
    segment_ids.append(0)

    for token in tokens:
        input_tokens.append(token)
        segment_ids.append(0)

    input_tokens.append("[SEP]")
    segment_ids.append(0)

    input_ids = tokenizer.convert_tokens_to_ids(input_tokens)

    # The mask has 1 for real tokens and 0 for padding tokens. Only real tokens are attended to.
    input_masks = [1] * len(input_ids)

    # Zero-pad up to the sequence length.
    while len(input_ids) < max_seq_length:
        input_ids.append(0)
        input_masks.append(0)
        segment_ids.append(0)

    assert len(input_ids) == max_seq_length
    assert len(input_masks) == max_seq_length
    assert len(segment_ids) == max_seq_length

    if ex_index < 5:
        tf.logging.info("*** Example ***")
        tf.logging.info("guid: %s" % (example.guid))
        tf.logging.info("tokens: %s" % " ".join([tokenization.printable_text(x) for x in tokens]))
        tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
        tf.logging.info("input_masks: %s" % " ".join([str(x) for x in input_masks]))
        tf.logging.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
        tf.logging.info("sent_label_id: %s" % str(sent_label_id))

    feature = InputFeatures(
        input_ids=input_ids,
        input_masks=input_masks,
        segment_ids=segment_ids,
        sent_label_id=sent_label_id)
    return feature

def convert_examples_to_features(examples,
                                 sent_label_list,
                                 max_seq_length,
                                 tokenizer):
    """Convert a set of `InputExample`s to a list of `InputFeatures`."""
    features = []
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            tf.logging.info("Writing example %d of %d" % (ex_index, len(examples)))

        feature = convert_single_example(ex_index, example, sent_label_list, max_seq_length, tokenizer)
        features.append(feature)

    return features

def input_fn_builder(features,
                     seq_length,
                     is_training,
                     drop_remainder):
    """Creates an `input_fn` closure to be passed to TPUEstimator."""
    all_input_ids = []
    all_input_masks = []
    all_segment_ids = []
    all_sent_label_ids = []

    for feature in features:
        all_input_ids.append(feature.input_ids)
        all_input_masks.append(feature.input_masks)
        all_segment_ids.append(feature.segment_ids)
        all_sent_label_ids.append(feature.sent_label_id)

    def input_fn(params):
        batch_size = params["batch_size"]
        num_examples = len(features)

        # This is for demo purposes and does NOT scale to large data sets. We do
        # not use Dataset.from_generator() because that uses tf.py_func which is
        # not TPU compatible. The right way to load data is with TFRecordReader.
        d = tf.data.Dataset.from_tensor_slices({
            "input_ids": tf.constant(all_input_ids, shape=[num_examples, seq_length], dtype=tf.int32),
            "input_masks": tf.constant(all_input_masks, shape=[num_examples, seq_length], dtype=tf.int32),
            "segment_ids": tf.constant(all_segment_ids, shape=[num_examples, seq_length], dtype=tf.int32),
            "sent_label_ids": tf.constant(all_sent_label_ids, shape=[num_examples], dtype=tf.int32),
        })

        if is_training:
            d = d.repeat()
            d = d.shuffle(buffer_size=100, seed=np.random.randint(10000))

        d = d.batch(batch_size=batch_size, drop_remainder=drop_remainder)
        return d

    return input_fn

def file_based_convert_examples_to_features(examples,
                                            sent_label_list,
                                            max_seq_length,
                                            tokenizer,
                                            output_file):
    """Convert a set of `InputExample`s to a TFRecord file."""
    def create_int_feature(values):
        return tf.train.Feature(int64_list=tf.train.Int64List(value=list(values)))

    writer = tf.python_io.TFRecordWriter(output_file)

    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            tf.logging.info("Writing example %d of %d" % (ex_index, len(examples)))

        feature = convert_single_example(ex_index, example, sent_label_list, max_seq_length, tokenizer)

        features = collections.OrderedDict()
        features["input_ids"] = create_int_feature(feature.input_ids)
        features["input_masks"] = create_int_feature(feature.input_masks)
        features["segment_ids"] = create_int_feature(feature.segment_ids)
        features["sent_label_ids"] = create_int_feature([feature.sent_label_id])

        tf_example = tf.train.Example(features=tf.train.Features(feature=features))

        writer.write(tf_example.SerializeToString())

    writer.close()

def file_based_input_fn_builder(input_file,
                                seq_length,
                                is_training,
                                drop_remainder):
    """Creates an `input_fn` closure to be passed to TPUEstimator."""
    name_to_features = {
        "input_ids": tf.FixedLenFeature([seq_length], tf.int64),
        "input_masks": tf.FixedLenFeature([seq_length], tf.int64),
        "segment_ids": tf.FixedLenFeature([seq_length], tf.int64),
        "sent_label_ids": tf.FixedLenFeature([], tf.int64),
    }

    def _decode_record(record,
                       name_to_features):
        """Decodes a record to a TensorFlow example."""
        example = tf.parse_single_example(record, name_to_features)

        # tf.Example only supports tf.int64, but the TPU only supports tf.int32. So cast all int64 to int32.
        for name in list(example.keys()):
            t = example[name]
            if t.dtype == tf.int64:
                t = tf.to_int32(t)
            example[name] = t

        return example

    def input_fn(params):
        """The actual input function."""
        batch_size = params["batch_size"]

        # For training, we want a lot of parallel reading and shuffling.
        # For eval, we want no shuffling and parallel reading doesn't matter.
        d = tf.data.TFRecordDataset(input_file)

        if is_training:
            d = d.repeat()
            d = d.shuffle(buffer_size=100, seed=np.random.randint(10000))

        d = d.apply(tf.contrib.data.map_and_batch(
            lambda record: _decode_record(record, name_to_features),
            batch_size=batch_size,
            drop_remainder=drop_remainder))

        return d

    return input_fn

def create_model(bert_config,
                 input_ids,
                 input_masks,
                 segment_ids,
                 sent_label_ids,
                 sent_label_list,
                 mode,
                 use_tpu):
    """Creates a Classifier model."""
    is_training = (mode == tf.estimator.ModeKeys.TRAIN)
    model = modeling.BertModel(
        config=bert_config,
        is_training=is_training,
        input_ids=input_ids,
        input_mask=input_masks,
        token_type_ids=segment_ids,
        use_one_hot_embeddings=use_tpu)

    # If you want to use sentence-level output, use model.get_pooled_output()
    # If you want to use token-level output, use model.get_sequence_output()
    with tf.variable_scope("sent", reuse=tf.AUTO_REUSE):
        sent_result = model.get_pooled_output()
        sent_result_mask = tf.cast(tf.reduce_max(input_masks, axis=-1, keepdims=True), dtype=tf.float32)

        sent_kernel_initializer = tf.glorot_uniform_initializer(seed=np.random.randint(10000), dtype=tf.float32)
        sent_bias_initializer = tf.zeros_initializer
        sent_dense_layer = tf.keras.layers.Dense(units=len(sent_label_list), activation=None, use_bias=True,
            kernel_initializer=sent_kernel_initializer, bias_initializer=sent_bias_initializer,
            kernel_regularizer=None, bias_regularizer=None, trainable=True)

        sent_dropout_layer = tf.keras.layers.Dropout(rate=0.1, seed=np.random.randint(10000))

        sent_result = sent_dense_layer(sent_result)
        if mode == tf.estimator.ModeKeys.TRAIN:
            sent_result = sent_dropout_layer(sent_result)

        masked_sent_predict = sent_result * sent_result_mask + MIN_FLOAT * (1 - sent_result_mask)
        sent_predict_probs = tf.nn.softmax(masked_sent_predict, axis=-1)
        sent_predict_ids = tf.cast(tf.argmax(sent_predict_probs, axis=-1), dtype=tf.int32)
        sent_predict_scores = tf.reduce_max(sent_predict_probs, axis=-1)

    loss = tf.constant(0.0, dtype=tf.float32)
    if mode not in [tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL]:
        return loss, sent_predict_ids, sent_predict_scores, sent_predict_probs

    if sent_label_ids is not None:
        with tf.variable_scope("sent_loss", reuse=tf.AUTO_REUSE):
            sent_label = tf.cast(sent_label_ids, dtype=tf.float32)
            sent_label_mask = tf.cast(tf.reduce_max(input_masks, axis=-1), dtype=tf.float32)
            masked_sent_label = tf.cast(sent_label * sent_label_mask, dtype=tf.int32)
            sent_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=masked_sent_label, logits=masked_sent_predict)
            sent_loss = tf.reduce_sum(sent_cross_entropy * sent_label_mask) / tf.reduce_sum(tf.reduce_max(sent_label_mask, axis=-1))
            loss = loss + sent_loss

    return loss, sent_predict_ids, sent_predict_scores, sent_predict_probs

def model_fn_builder(bert_config,
                     sent_label_list,
                     init_checkpoint,
                     learning_rate,
                     num_train_steps,
                     num_warmup_steps,
                     use_tpu):
    """Returns `model_fn` closure for TPUEstimator."""
    def model_fn(features,
                 labels,
                 mode,
                 params):  # pylint: disable=unused-argument
        """The `model_fn` for TPUEstimator."""
        tf.logging.info("*** Features ***")
        for name in sorted(features.keys()):
            tf.logging.info("  name = %s, shape = %s" % (name, features[name].shape))

        input_ids = features["input_ids"]
        input_masks = features["input_masks"]
        segment_ids = features["segment_ids"]
        sent_label_ids = features["sent_label_ids"] if mode in [tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL] else None

        loss, sent_predict_ids, sent_predict_scores, sent_predict_probs = create_model(bert_config,
            input_ids, input_masks, segment_ids, sent_label_ids, sent_label_list, mode, use_tpu)

        tvars = tf.trainable_variables()
        initialized_variable_names = {}
        scaffold_fn = None

        if init_checkpoint:
            assignment_map, initialized_variable_names = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)

        if use_tpu:
            def tpu_scaffold():
                tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
                return tf.train.Scaffold()

            scaffold_fn = tpu_scaffold
        else:
            tf.train.init_from_checkpoint(init_checkpoint, assignment_map)

        tf.logging.info("**** Trainable Variables ****")
        for var in tvars:
            init_string = ""
            if var.name in initialized_variable_names:
                init_string = ", *INIT_FROM_CKPT*"

            tf.logging.info("  name = %s, shape = %s%s", var.name, var.shape, init_string)

        output_spec = None
        if mode == tf.estimator.ModeKeys.TRAIN:
            train_op = optimization.create_optimizer(loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu)
            output_spec = tf.contrib.tpu.TPUEstimatorSpec(
                mode=mode,
                loss=loss,
                train_op=train_op,
                scaffold_fn=scaffold_fn)
        elif mode == tf.estimator.ModeKeys.EVAL:
            def metric_fn(sent_label_ids,
                          sent_predict_ids):
                sent_accuracy = tf.metrics.accuracy(labels=sent_label_ids, predictions=sent_predict_ids)

                metric = {
                    "sent_accuracy": sent_accuracy,
                }

                return metric

            eval_metrics = (metric_fn, [sent_label_ids, sent_predict_ids])
            output_spec = tf.contrib.tpu.TPUEstimatorSpec(
                mode=mode,
                loss=loss,
                eval_metrics=eval_metrics,
                scaffold_fn=scaffold_fn)
        else:
            output_spec = tf.contrib.tpu.TPUEstimatorSpec(
                mode=mode,
                predictions={
                    "sent_predict_id": sent_predict_ids,
                    "sent_predict_score": sent_predict_scores,
                    "sent_predict_prob": sent_predict_probs
                },
                scaffold_fn=scaffold_fn)

        return output_spec

    return model_fn

def get_masked_data(data_ids,
                    label_list):
    label_map = {}
    for (i, label) in enumerate(label_list):
        label_map[label] = i

    pad_id = tf.constant(label_map["[PAD]"], shape=[], dtype=tf.int32)
    out_id = tf.constant(label_map["O"], shape=[], dtype=tf.int32)
    x_id = tf.constant(label_map["X"], shape=[], dtype=tf.int32)
    cls_id = tf.constant(label_map["[CLS]"], shape=[], dtype=tf.int32)
    sep_id = tf.constant(label_map["[SEP]"], shape=[], dtype=tf.int32)

    masked_data_ids = (tf.cast(tf.not_equal(data_ids, pad_id), dtype=tf.int32) *
        tf.cast(tf.not_equal(data_ids, out_id), dtype=tf.int32) *
        tf.cast(tf.not_equal(data_ids, x_id), dtype=tf.int32) *
        tf.cast(tf.not_equal(data_ids, cls_id), dtype=tf.int32) *
        tf.cast(tf.not_equal(data_ids, sep_id), dtype=tf.int32))

    return masked_data_ids

def serving_input_fn():
    with tf.variable_scope("export"):
        features = {
            'input_ids': tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='input_ids'),
            'input_masks': tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='input_masks'),
            'segment_ids': tf.placeholder(tf.int32, [None, FLAGS.max_seq_length], name='segment_ids')
        }

        return tf.estimator.export.build_raw_serving_input_receiver_fn(features)()

def decode_predicts(predicts,
                    sent_label_list,
                    max_seq_length,
                    tokenizer):
    decoded_predicts = []
    for predict in predicts:
        input_tokens = tokenizer.convert_ids_to_tokens(predict["input_ids"])
        input_masks = predict["input_masks"]

        decoded_tokens = []
        for input_token, input_mask in zip(input_tokens, input_masks):
            if input_mask == 0:
                break

            if input_token in ["[CLS]", "[SEP]"]:
                continue

            if input_token[:2] == "##":
                decoded_tokens[-1] = decoded_tokens[-1] + input_token[2:]
                continue

            decoded_tokens.append(input_token)

        decoded_predict = {
            "text": " ".join(decoded_tokens),
            "sent_label": sent_label_list[predict["sent_label_id"]],
            "sent_predict": sent_label_list[predict["sent_predict_id"]],
            "sent_score": float(predict["sent_predict_score"]),
            "sent_probs": [float(prob) for prob in predict["sent_predict_prob"]]
        }

        decoded_predicts.append(decoded_predict)

    return decoded_predicts

def write_to_json(data_list,
                  data_path):
    data_folder = os.path.dirname(data_path)
    if not os.path.exists(data_folder):
        os.mkdir(data_folder)

    with open(data_path, "w") as file:
        json.dump(data_list, file, indent=4)

def write_to_text(data_list,
                  data_path):
    data_folder = os.path.dirname(data_path)
    if not os.path.exists(data_folder):
        os.mkdir(data_folder)

    with open(data_path, "w") as file:
        for data in data_list:
            file.write("{0}\n".format(data))

def main(_):
    tf.logging.set_verbosity(tf.logging.INFO)

    np.random.seed(FLAGS.random_seed)

    bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)

    if FLAGS.max_seq_length > bert_config.max_position_embeddings:
        raise ValueError("Cannot use sequence length %d because the BERT model was only trained up to sequence length %d" %
            (FLAGS.max_seq_length, bert_config.max_position_embeddings))

    tf.gfile.MakeDirs(FLAGS.output_dir)

    tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case, FLAGS.init_checkpoint)
    tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)

    data_dir = FLAGS.data_dir
    task_name = FLAGS.task_name.lower()
    processor = ClassificationProcessor(data_dir, task_name)
    sent_label_list = processor.get_sent_labels()

    train_examples = None
    num_train_steps = None
    num_warmup_steps = None
    if FLAGS.do_train:
        train_examples = processor.get_train_examples()
        num_train_steps = int(len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs)
        num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)

    tpu_cluster_resolver = None
    if FLAGS.use_tpu and FLAGS.tpu_name:
        tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
            FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)

    is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
    run_config = tf.contrib.tpu.RunConfig(
        cluster=tpu_cluster_resolver,
        master=FLAGS.master,
        model_dir=FLAGS.output_dir,
        save_checkpoints_steps=FLAGS.save_checkpoints_steps,
        tpu_config=tf.contrib.tpu.TPUConfig(
            iterations_per_loop=FLAGS.iterations_per_loop,
            num_shards=FLAGS.num_tpu_cores,
            per_host_input_for_training=is_per_host))

    model_fn = model_fn_builder(
        bert_config=bert_config,
        sent_label_list=sent_label_list,
        init_checkpoint=FLAGS.init_checkpoint,
        learning_rate=FLAGS.learning_rate,
        num_train_steps=num_train_steps,
        num_warmup_steps=num_warmup_steps,
        use_tpu=FLAGS.use_tpu)

    # If TPU is not available, this will fall back to normal Estimator on CPU or GPU.
    estimator = tf.contrib.tpu.TPUEstimator(
        use_tpu=FLAGS.use_tpu,
        model_fn=model_fn,
        config=run_config,
        export_to_tpu=FLAGS.use_tpu,
        train_batch_size=FLAGS.train_batch_size,
        eval_batch_size=FLAGS.eval_batch_size,
        predict_batch_size=FLAGS.predict_batch_size)

    if FLAGS.do_train:
        tf.logging.info("***** Run training *****")
        tf.logging.info("  Num examples = %d", len(train_examples))
        tf.logging.info("  Batch size = %d", FLAGS.train_batch_size)
        tf.logging.info("  Num steps = %d", num_train_steps)

        train_features = convert_examples_to_features(
            examples=train_examples,
            sent_label_list=sent_label_list,
            max_seq_length=FLAGS.max_seq_length,
            tokenizer=tokenizer)

        train_input_fn = input_fn_builder(
            features=train_features,
            seq_length=FLAGS.max_seq_length,
            is_training=True,
            drop_remainder=True)

        estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)

    if FLAGS.do_eval:
        eval_examples = processor.get_dev_examples()
        tf.logging.info("***** Run evaluation *****")
        tf.logging.info("  Num examples = %d", len(eval_examples))
        tf.logging.info("  Batch size = %d", FLAGS.eval_batch_size)

        eval_features = convert_examples_to_features(
            examples=eval_examples,
            sent_label_list=sent_label_list,
            max_seq_length=FLAGS.max_seq_length,
            tokenizer=tokenizer)

        eval_input_fn = input_fn_builder(
            features=eval_features,
            seq_length=FLAGS.max_seq_length,
            is_training=False,
            drop_remainder=False)

        result = estimator.evaluate(input_fn=eval_input_fn)

        sent_accuracy = result["sent_accuracy"]

        tf.logging.info("***** Evaluation result *****")
        tf.logging.info("  Accuracy (sent-level) = %s", str(sent_accuracy))

    if FLAGS.do_predict:
        predict_examples = processor.get_test_examples()
        tf.logging.info("***** Run prediction *****")
        tf.logging.info("  Num examples = %d", len(predict_examples))
        tf.logging.info("  Batch size = %d", FLAGS.predict_batch_size)

        predict_features = convert_examples_to_features(
            examples=predict_examples,
            sent_label_list=sent_label_list,
            max_seq_length=FLAGS.max_seq_length,
            tokenizer=tokenizer)

        predict_input_fn = input_fn_builder(
            features=predict_features,
            seq_length=FLAGS.max_seq_length,
            is_training=False,
            drop_remainder=False)

        result = estimator.predict(input_fn=predict_input_fn)

        predicts = [{
            "input_ids": feature.input_ids,
            "input_masks": feature.input_masks,
            "sent_label_id": feature.sent_label_id,
            "sent_predict_id": predict["sent_predict_id"],
            "sent_predict_score": predict["sent_predict_score"],
            "sent_predict_prob": predict["sent_predict_prob"].tolist()
        } for feature, predict in zip(predict_features, result)]

        decoded_predicts = decode_predicts(
            predicts=predicts,
            sent_label_list=sent_label_list,
            max_seq_length=FLAGS.max_seq_length,
            tokenizer=tokenizer)

        predict_tag = FLAGS.predict_tag if FLAGS.predict_tag else str(time.time())
        output_path = os.path.join(FLAGS.output_dir, "predict.{0}.json".format(predict_tag))
        write_to_json(decoded_predicts, output_path)

    if FLAGS.do_export:
        tf.logging.info("***** Running exporting *****")
        tf.gfile.MakeDirs(FLAGS.export_dir)
        estimator.export_savedmodel(FLAGS.export_dir, serving_input_fn, as_text=False)

if __name__ == "__main__":
    flags.mark_flag_as_required("bert_config_file")
    flags.mark_flag_as_required("vocab_file")
    flags.mark_flag_as_required("init_checkpoint")
    flags.mark_flag_as_required("data_dir")
    flags.mark_flag_as_required("output_dir")
    flags.mark_flag_as_required("export_dir")
    tf.app.run()


from mtcnn.mtcnn import MTCNN

class mtcnn_to_face_alignment:
    def __init__(self):
        self.detector = MTCNN()

    def find_bboxes(self, input_img):
        """
        Recieve: an image

        Return:  list of bbox format (x1, y1, x2, y2)
        """
        faces_positions = self.detector.detect_faces(input_img)
        #format: (x1, y1, w, h) y probabilidad pero no interesa
        bbox_list = []
        if (len(faces_positions)!=0):
            for face in faces_positions:
                bbox = face['box']
                bbox_formated = [[bbox[0], bbox[1], bbox[0]+bbox[2], bbox[1]+bbox[3]]]
                bbox_list.append(bbox_formated)
        return bbox_list

    def size_of_bbox(self, bbox):
        """
        return:
            multiplication of width x height
        """
        return bbox[2]*bbox[3]
    #terminar
    def mtcnn_bbox_face_alignment_format_a(self, input_img):
        """
        Recieve: an image

        Return: the biggest bbox format (x1, y1, x2, y2)
        """
        max_size_box = 0
        size_box = 0
        bbox_to_return = None
        faces_positions = self.detector.detect_faces(input_img)
        #format: (x1, y1, w, h) y probabilidad pero no interesa
        bbox_list = []
        if (len(faces_positions)!=0):
            for face in faces_positions:
                bbox = face['box']
                size_box = size_of_bbox(bbox)
                if (size_box > max_size_box):
                    bbox_to_return = bbox
            ###terminar para bbox return  en nuevo formato
            bbox = bbox_to_return
            bbox_formated = [[bbox[0], bbox[1], bbox[0]+bbox[2], bbox[1]+bbox[3]]]
            bbox_list.append(bbox_formated)
        return bbox_list


from django.conf.urls import patterns, include, url
from settings import APP_ROOT

urlpatterns = patterns('',
    url(r'^$', 'register_users.views.home', name='home'),

    url(r'^static/(?P<path>.*)$', 'django.views.static.serve', {'document_root': APP_ROOT+'static/'}),

    url(r'^uploads/(?P<path>.*)$', 'django.views.static.serve', {'document_root': APP_ROOT+'uploads/'}),

    url(r'^media/img/(?P<path>.*)$', 'django.views.static.serve', {'document_root': APP_ROOT+'uploads/photos/'}, name='images_link'),

    url(r'^accounts/', include('register_users.urls')),

    url(r'^msg/', include('private_message.urls')),

    url(r'^friend/', include('friends_app.urls')),

    url(r'^media/', include('media_app.urls')),

    url(r'^music/', include('music_app.urls')),

    url(r'^settings/', include('settings_app.urls')),

    url(r'^find/', include('find_app.urls'))
)


from django.contrib import admin
from tpo.app.models import UserProfile, Company

class UserProfileAdmin(admin.ModelAdmin):
    fields = ('name', 'username', 'email',)

class CompanyAdmin(admin.ModelAdmin):
    search_fields = ('name', 'tpr__name', 'tpr__email', 'tpr__username')
    list_display = ('name', 'tpr',)
    list_filter = ('type_of_company',)

admin.site.register(UserProfile, UserProfileAdmin)
admin.site.register(Company, CompanyAdmin)


# coding: utf-8

import urllib2
import urllib
import urlparse
from zope.component import getUtility
from plone.registry.interfaces import IRegistry
from collective.socialpublish.controlpanel.interfaces import ISocialPublishControlPanel
from Products.Five.browser import BrowserView
#from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile
from Products.CMFCore.utils import getToolByName
from Products.statusmessages.interfaces import IStatusMessage

from collective.socialpublish.events import get_page_list
from collective.socialpublish.controlpanel.utils import fb_page_info_list_to_str

ENDPOINT = 'graph.facebook.com'

def get_url(path, args=None):
    args = args or {}
    #if ACCESS_TOKEN:
    #    args['access_token'] = ACCESS_TOKEN
    if 'access_token' in args or 'client_secret' in args:
        endpoint = "https://" + ENDPOINT
    else:
        endpoint = "http://" + ENDPOINT
    return endpoint + path + '?' + urllib.urlencode(args)

def get_resource(path, args=None):
    return urllib2.urlopen(get_url(path, args=args)).read()

class FacebookAuth(BrowserView):
    """

    """
    #template = ViewPageTemplateFile('')

    def __call__(self):
        portal_messages = IStatusMessage(self.request)
        portal_url = getToolByName(self.context, 'portal_url')()
        here_url = portal_url + "/@@facebook-auth"
        registry = getUtility(IRegistry)
        settings = registry.forInterface(ISocialPublishControlPanel)
        fb_app_id = settings.fb_app_id
        fb_app_secret = settings.fb_app_secret

        code = self.request.form.get('code')
        if code is None:
            portal_messages.add(u"Illegal access because nothing 'code'", type=u"error")
            return self.request.RESPONSE.redirect("@@socialpublish-settings")
        token_res = get_resource('/oauth/access_token', {'client_id': fb_app_id,
                                               'redirect_uri': here_url,
                                               'client_secret': fb_app_secret,
                                               'code': code})
        fb_access_token = urlparse.parse_qs(token_res).get('access_token')
        try:
            fb_access_token_unicode = unicode(fb_access_token[0], 'utf-8')
        except IndexError:
            portal_messages.add(u"Couldn't get Facebook token", type=u"error")
            return self.request.RESPONSE.redirect("@@socialpublish-settings")
        settings.fb_access_token = fb_access_token_unicode

        page_info_list = get_page_list(fb_access_token_unicode)
        settings.fb_page_info = fb_page_info_list_to_str(page_info_list)

        portal_messages.add(u"Getting Facebook page information", type=u"info")
        return self.request.RESPONSE.redirect("@@socialpublish-settings")


import pyfits
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import fsolve
from astropy.io import fits

data1 = pyfits.getdata('4imAF.fits')
data2 = pyfits.getdata('3imAF.fits')

#set a noise floor for the image. Poor SNR pixels will give you bad temperature measurements.
floor1 = 0.045
floor2 = 0.070

#set low SNR pixels to nans
av1 = np.where(data1 < floor1)
data1[av1] = np.nan

#set low SNR pixels to nans
av2 = np.where(data2 < floor2)
data2[av2] = np.nan

#Color temperatures

#extinction factor for the images (provided by Matt)
bg1 = 2.58
bg2 = 1.54

#Bg from Fit3.py code, aka. fitting the histogram
        #I_1Bg = Band 3 background Intensity
        #I_2Bg = Band 4 background Intensity
I_1Bg = 325
I_2Bg = 1261

#wavelengths of first image in cm for cgs
l1 = 12e-4
l2 = 22e-4

#set constants (in cgs)
c = 3e10
h = 6.626e-27
c = 2.9979e10
kb = 1.38e-16

#convert wavelengths of images to frequencies
v1 = c / l1
v2 = c / l2

#define function for b_nu that we will solve for T
func = lambda T : (v1/v2)**5*(np.expm1((h*v2)/(kb*T)))/(np.expm1((h*v1)/(kb*T)))-I_1/I_2

#provide an initial guess for T for the solver to use.
Tinit = 100.0

#get the shape of the images so we know how big to make the for loop
shp = np.shape(data1)

#create a new empty array to store the temperature values
Tim = np.zeros((shp[0],shp[1]))

for i  in range (0,shp[0]):
    for j in range (0,shp[1]):

        #store the pixel values from the images
        I_1 = data1[i,j]
        I_2 = data2[i,j]

        #apply the extinction factors
        I_1 = I_1*bg1
        I_2 = I_2*bg2

        #use the solver to find the temperatures
        TS = fsolve(func,Tinit)

        #store the values in the array
        Tim[i,j] = TS

#Where the solver returned Tinit, turn into a nan.
Tim[Tim == Tinit] = np.nan

#Creating a new array
hdu = fits.PrimaryHDU(Tim)
hdulist = fits.HDUList([hdu])
hdulist.writeto('1_ColorTempAF_5.fits')

#plot the color temperature map
plt.figure()
plt.title('Color-Tempature Map')
plt.imshow(Tim[::-1])
#plt.clim(lowval, highval)
plt.colorbar()
plt.show()

#fits.writeto('out.fits', data, header)


# def bubble_sort(l):
#     length = len(l)
#     for i in range(length):
#         try:
#             for j in range(length-i):
#                 if l[j] > l[j+1]:
#                     tmp = l[j+1]
#                     l[j+1] = l[j]
#                     l[j] = tmp
#         except IndexError:
#             pass
#
#     return l

#变量其实是地址 # 101011100111101011 #逻辑地址  -->物理地址 01111110001100
import datetime

def get_time():
    return datetime.time()

def f2():
    now = get_time()

def select_sort(l):

    now = get_time()

    length = len(l)
    for i in range(length):
        little = l[i]
        min_index = i+1
        for j in range(i+1,length):
            if l[j] < l[min_index]:
                min_index = j
        try:
            if l[min_index] < little:
                tmp = little
                l[i] = l[min_index]
                l[min_index] = tmp
        except IndexError:
            pass

    return l

a1 = [1,2,10,3,4,5,9,56,6,7,134,9]

# print(select_sort(a1))

def muti_seli():
    sql = 'select 1;'
    sql2 = 'select 2;'

x = None
x12 = 'None'

str1 = "\\r"

print(str1)

print('-------')


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /Users/breno/Envs/djangoplus/lib/python3.7/site-packages/djangoplus/utils/dateutils.py
# Compiled at: 2019-04-13 18:22:56
# Size of source mod 2**32: 2346 bytes
import datetime, calendar
DAY_NAMES = [calendar.day_name[i].capitalize().split()[0] for i in range(0, 7)]
DAY_INITIALS = [calendar.day_name[i][0:3].capitalize() for i in range(0, 7)]
MONTH_NAMES = [calendar.month_name[i].capitalize() for i in range(1, 13)]
MONTH_INITIALS = [calendar.month_name[i][0:3].capitalize() for i in range(1, 13)]
DAY_NAMES_CHOICES = [[x, x] for x in DAY_NAMES]
DAY_INITIALS_CHOICES = [[x, x] for x in DAY_INITIALS]
MONTH_NAMES_CHOICES = [[x, x] for x in MONTH_NAMES]
MONTH_INITIALS_CHOICES = [[x, x] for x in MONTH_INITIALS]

def calculate_age(birthday):
    today = datetime.date.today()
    return today.year - birthday.year - ((today.month, today.day) < (birthday.month, birthday.day))

def numer_of_days(start, end):
    delta = end - start
    return delta.days

def parse_date(date_string):
    if len(date_string) == 10:
        fmt = '%d/%m/%Y'
    else:
        fmt = '%d/%m/%Y %H:%M:%S'
    return datetime.datetime.strptime(date_string, fmt)

def add_months(sourcedate, months):
    month = sourcedate.month - 1 + months
    year = int(sourcedate.year + month / 12)
    month = month % 12 + 1
    day = min(sourcedate.day, calendar.monthrange(year, month)[1])
    return datetime.date(year, month, day)

def add_days(sourcedate, days):
    sourcedate = sourcedate or datetime.date.today()
    return sourcedate + datetime.timedelta(days=days)

def future(days):
    return datetime.date.today() + datetime.timedelta(days=days)

def past(days):
    return datetime.date.today() - datetime.timedelta(days=days)

def pretty_date(d):
    diff = datetime.datetime.now() - d
    s = diff.seconds
    if diff.days > 365 or diff.days < 0:
        return d.strftime('%d %b %y')
    if 60 > diff.days > 30:
        return '1 mês atrás'
    if diff.days > 60:
        return '{} meses atrás'.format(diff.days / 30)
    if diff.days == 1:
        return '1 dia atrás'
    if diff.days > 1:
        return '{} dias atrás'.format(diff.days)
    if s <= 1:
        return 'agora'
    if s < 60:
        return '{} segundos atrás'.format(s)
    if s < 120:
        return '1 minuto atrás'
    if s < 3600:
        return '{} minutos atrás'.format(s / 60)
    if s < 7200:
        return '1 hora atrás'
    return '{} horas atrás'.format(s / 3600)


import os
import sys
import traceback
from unittest import mock

from click.testing import CliRunner

from .mock_tables import dbconnector

import config.main as config
import show.main as show
from utilities_common.db import Db

show_interfaces_mpls_output="""\
Interface     MPLS State
------------  ------------
Ethernet2     enable
Ethernet4     disable
Ethernet8     disable
Ethernet16    disable
Loopback0     disable
PortChannel2  disable
Vlan2         enable
"""

show_interfaces_mpls_specific_output="""\
Interface    MPLS State
-----------  ------------
Ethernet2    enable
"""

modules_path = os.path.join(os.path.dirname(__file__), "..")
test_path = os.path.join(modules_path, "tests")
sys.path.insert(0, modules_path)
sys.path.insert(0, test_path)
mock_db_path = os.path.join(test_path, "mpls_input")

class TestMpls(object):
    @classmethod
    def setup_class(cls):
        print("SETUP")
        os.environ['UTILITIES_UNIT_TESTING'] = "1"

    def test_config_mpls_add(self):
        runner = CliRunner()
        db = Db()
        obj = {'config_db':db.cfgdb}

        result = runner.invoke(config.config.commands["interface"].commands["mpls"].commands["add"], ["Ethernet4"], obj=obj)
        print(result.exit_code)
        print(result.output)
        assert result.exit_code == 0
        assert db.cfgdb.get_entry("INTERFACE", "Ethernet4") == {"mpls": "enable"}

    def test_config_mpls_remove(self):
        runner = CliRunner()
        db = Db()
        obj = {'config_db':db.cfgdb}

        result = runner.invoke(config.config.commands["interface"].commands["mpls"].commands["remove"], ["Ethernet4"], obj=obj)
        print(result.exit_code)
        print(result.output)
        assert result.exit_code == 0
        assert db.cfgdb.get_entry("INTERFACE", "Ethernet4") == {"mpls": "disable"}

    def test_show_interfaces_mpls(self):
        jsonfile = os.path.join(mock_db_path, 'appl_db')
        dbconnector.dedicated_dbs['APPL_DB'] = jsonfile

        runner = CliRunner()
        result = runner.invoke(show.cli.commands["interfaces"].commands["mpls"], [])
        print(result.exit_code)
        print(result.output)
        assert result.exit_code == 0
        assert result.output == show_interfaces_mpls_output

    def test_show_interfaces_mpls_specific(self):
        jsonfile = os.path.join(mock_db_path, 'appl_db')
        dbconnector.dedicated_dbs['APPL_DB'] = jsonfile

        runner = CliRunner()
        result = runner.invoke(show.cli.commands["interfaces"].commands["mpls"], ["Ethernet2"])
        print(result.exit_code)
        print(result.output)
        assert result.exit_code == 0
        assert result.output == show_interfaces_mpls_specific_output

    @classmethod
    def teardown_class(cls):
        print("TEARDOWN")
        os.environ['UTILITIES_UNIT_TESTING'] = "0"
        dbconnector.dedicated_dbs['APPL_DB'] = None


"""
下载
"""
from __future__ import (absolute_import, division,
                        print_function, unicode_literals)

from urllib.request import urlopen

import json
import requests

json_url = "https://raw.githubusercontent.com/muxuezi/btc/master/btc_close_2017.json"
response = urlopen(json_url)

# 读取数据
req = response.read()
# print(type(req))
# print(req)
# 将数据写入文件
with open("btc_close_2017_down.json", 'wb') as f:
    f.write(req)

# 加载json格式
# with open("btc_close_2017_down.json", 'r') as f:
#     file_urllib = json.load(f)
#     print(file_urllib)
file_urllib = json.loads(str(req, 'utf-8'))
print(file_urllib)

# 使用requests

req = requests.get(json_url)
with open('btc_close_2017_down_requests.json', 'w') as f:
    f.write(req.text)

print(req.json())


import json
import logging
from dataclasses import asdict
from json.decoder import JSONDecodeError

import asyncclick as click
import trio
from trio_websocket import ConnectionClosed, serve_websocket

from utils import Bus, Frame, WindowBounds
from validators import get_validated_bus_data, get_validated_map_frame

BUSES = {}

async def start_bus_data_share(request):
    """
    Accepts: json with "busId","lat","lng","route"
    """
    ws = await request.accept()
    while True:
        try:
            bus_json = await ws.get_message()
            bus_dict = get_validated_bus_data(bus_json)
            if bus_dict is None:
                continue
            else:
                await ws.send_message("ok")
            bus_id = bus_dict["busId"]
            BUSES[bus_id] = Bus(**bus_dict)
            logging.info(f"got {bus_id}")
        except ValueError as e:
            logging.info(f"bus = {e}")
            await ws.send_message(
                json.dumps({"bus_error": str(e), "msgType": "Error"})
            )
        except JSONDecodeError:
            logging.info("Invalid JSON Data")
            await ws.send_message(
                '{"bus_error": "Invalid JSON Data", "msgType": "Error"}'
            )

async def start_browser_data_share(request):
    """
    Sends: buses json
    Receives: Browser Frame with coordinatess
    """
    frame = Frame()
    ws = await request.accept()
    async with trio.open_nursery() as nursery:
        nursery.start_soon(get_map_frame, ws, frame)
        nursery.start_soon(send_buses_data, ws, frame)

async def send_buses_data(ws, frame: Frame):
    while True:
        logging.info("got browser request")
        if frame.get_bounds() is not None:
            filtered_buses = filter(frame.is_inside, BUSES.values())

            await ws.send_message(
                json.dumps(
                    {
                        "msgType": "Buses",
                        "buses": [asdict(bus) for bus in filtered_buses],
                    }
                )
            )
        else:
            logging.info("frame is empty")
        await trio.sleep(2.5)

async def get_map_frame(ws, frame: Frame):
    while True:
        try:
            frame_json = await ws.get_message()
            logging.info("got frame from browser")
            frame_dict = get_validated_map_frame(frame_json)
            if frame_dict is None or (not "data" in frame_dict):
                continue
            frame.update(WindowBounds(**frame_dict["data"]))
        except ValueError as e:
            logging.info(f"map = {e}")
            await ws.send_message(
                json.dumps({"frame_error": str(e), "msgType": "Error"})
            )
        except JSONDecodeError:
            logging.info("Invalid JSON Data")
            await ws.send_message(
                '{"frame_error": "Invalid JSON Data", "msgType": "Error"}'
            )

async def bus_connection(port):
    while True:
        try:
            async with trio.open_nursery() as nursery:
                nursery.start_soon(
                    serve_websocket,
                    start_bus_data_share,
                    "127.0.0.1",
                    port,
                    None,
                )
        except ConnectionClosed:
            logging.info("sensors connection problem")

async def browser_connection(port):
    while True:
        try:
            async with trio.open_nursery() as nursery:
                nursery.start_soon(
                    serve_websocket,
                    start_browser_data_share,
                    "127.0.0.1",
                    port,
                    None,
                )
        except ConnectionClosed:
            logging.info("closed browser")

async def main(bus_port: int, browser_port: int, v: bool):
    if v:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.ERROR)

    async with trio.open_nursery() as nursery:
        nursery.start_soon(bus_connection, bus_port)
        nursery.start_soon(browser_connection, browser_port)

@click.command()
@click.option("--bus_port", default=8080, help="порт для имитатора автобусов")
@click.option("--browser_port", default=8000, help=" порт для браузера")
@click.option("--v", default=False, help="настройка логирования")
def run(bus_port: int, browser_port: int, v: bool):
    trio.run(main, bus_port, browser_port, v)

if __name__ == "__main__":
    run()


""" The entry point of configure and launch experiment.
"""
from exptools.launching.variant import VariantLevel, make_variants, update_config
from exptools.launching.affinity import encode_affinity, quick_affinity_code
from exptools.launching.exp_launcher import run_experiments

from os import path

from vos.experiments.launch_pretrain_vos import get_default_config

def main(args):
    experiment_title = "video_segmentation"
    affinity_code = encode_affinity(
        n_cpu_core= 48,
        n_gpu= 4,
        gpu_per_run= 4,
    )
    default_config = get_default_config()
    default_config["runner_kwargs"]["pretrain_optim_epochs"] = 0

    # set up variants
    variant_levels = list()

    values = [
        # ["EMN", ],
        ["STM", ],
    ]
    dir_names = ["NN{}".format(*v) for v in values]
    keys = [
        ("solution", ),
    ]
    variant_levels.append(VariantLevel(keys, values, dir_names))

    values = [
        # [1, 1, int(1e10), 0.9],
        [24, 24, 1e-5, int(1e10), 0.9],
        # [20,20,5e-5, int(1e10), 0.9],
    ]
    dir_names = ["train_spec-{}-{}-{}-{}".format(*v[1:]) for v in values]
    keys = [
        ("pretrain_dataloader_kwargs", "batch_size"),
        ("dataloader_kwargs", "batch_size"),
        ("algo_kwargs", "learning_rate"),
        ("algo_kwargs", "lr_max_iter"),
        ("algo_kwargs", "lr_power"),
    ]
    variant_levels.append(VariantLevel(keys, values, dir_names))

    values = [
        # [None],
        ["/root/VideoObjSeg/data/weightfiles/STM_pretrain_51.82-52.93.pkl"],
        # ["/root/VideoObjSeg/data/weightfiles/STM_fulltrain_62.84-66.74.pkl"],
        # ["/root/VideoObjSeg/data/weightfiles/EMN_pretrain_54.50-59.29.pkl"],
    ]
    dir_names = [("pretrainFalse" if i[0] is None else "pretrainTrue") for i in values]
    keys = [
        ("pretrain_snapshot_filename", ),
    ]
    variant_levels.append(VariantLevel(keys, values, dir_names))

    variants, log_dirs = make_variants(*variant_levels)
    for i, variant in enumerate(variants):
        variants[i] = update_config(default_config, variant)
        if args.debug > 0:
            # make sure each complete iteration has gone through and easy for debug
            variants[i]["runner_kwargs"]["pretrain_optim_epochs"] = 5
            variants[i]["runner_kwargs"]["max_optim_epochs"] = 5
            variants[i]["runner_kwargs"]["eval_interval"] = 2
            variants[i]["runner_kwargs"]["log_interval"] = 4
            variants[i]["pretrain_dataloader_kwargs"]["shuffle"] = False
            variants[i]["dataloader_kwargs"]["shuffle"] = False
            variants[i]["pretrain_dataloader_kwargs"]["num_workers"] = 0
            variants[i]["dataloader_kwargs"]["num_workers"] = 0
            variants[i]["eval_dataloader_kwargs"]["num_workers"] = 0
            variants[i]["random_subset_kwargs"]["subset_len"] = 2

    run_experiments(
        script="vos/experiments/videoSeg.py",
        affinity_code=affinity_code,
        experiment_title=experiment_title+("--debug" if args.debug else ""),
        runs_per_setting=1,
        variants=variants,
        log_dirs=log_dirs,
        debug_mode=args.debug,
    )

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()

    parser.add_argument(
        '--debug', help= 'A common setting of whether to entering debug mode for remote attach',
        type= int, default= 0,
    )

    args = parser.parse_args()
    if args.debug > 0:
        # configuration for remote attach and debug
        import ptvsd
        import sys
        ip_address = ('0.0.0.0', 5050)
        print("Process: " + " ".join(sys.argv[:]))
        print("Is waiting for attach at address: %s:%d" % ip_address, flush= True)
        # Allow other computers to attach to ptvsd at this IP address and port.
        ptvsd.enable_attach(address=ip_address,)
        # Pause the program until a remote debugger is attached
        ptvsd.wait_for_attach()
        print("Process attached, start running into experiment...", flush= True)
        ptvsd.break_into_debugger()

    main(args)


from flask import Response, request
from database.sucursal import Sucursal
from flask_restful import Resource

class SucursalApi(Resource):
    def get(self):
        agencies = Sucursal.objects().to_json()
        return Response(agencies, mimetype="application/json", status=200)

    def post(self):
        body = request.get_json()
        agency = Sucursal(**body).save()
        id = agency.id
        return {'id': str(id)}, 200


import os

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tango_with_django.settings')

import django
django.setup()
from rango.models import Category, Page

def populate():

    python_pages = [
        {"title": "Official Python Tutorial", "url": "http://docs.python.org"},
        {"title": "How to be a computer scientist", "url": "http://www.somebullshit.com"},
        {"title": "Learn Python in 10 mins", "url": "http://www.korokithakis.net"}
    ]

    django_pages = [
        {"title": "Official Django Tutorial", "url": "http://docs.djangoproject.com"},
        {"title": "Django Rocks", "url": "http://www.djangorocks.com"},
        {"title": "Tango with Djano", "url": "http://www.tangowithdjango.com"}
    ]

    other_pages = [
        {"title": "Volvic", "url": "http://www.volvic.com"},
        {"title": "Facebook", "url": "http://www.facebook.com"}
    ]

    cats = {"Python": {"pages": python_pages,"views": 128, "likes": 64}, "Django": {"pages": django_pages, "views": 64, "likes":32}, "Random": {"pages": other_pages, "views":32,"likes":16}}

    for cat, cat_data in cats.items():
        c = add_cat(cat, cat_data["views"],cat_data["likes"])
        for p in cat_data["pages"]:
            add_page(c, p["title"], p["url"])

    for c in Category.objects.all():
        for p in Page.objects.filter(category=c):
            print("- {0} - {1}".format(str(c), str(p)))

def add_page(cat, title, url, views=0):

    p = Page.objects.get_or_create(category=cat, title=title)[0]
    p.url=url
    p.views=views
    p.save()
    return p

def add_cat(name, views, likes):
    c = Category.objects.get_or_create(name=name)[0]
    c.views=views
    c.likes=likes
    c.save()
    return c

if __name__=='__main__':
    print ("Starting populate scrupt")
    populate()


from scipy.misc.pilutil import imresize

__author__ = 'peter'

def resize_while_preserving_aspect_ratio(im, x_dim=None, y_dim=None):
    """
    Resize an image, while preserving the aspect ratio.  For this you need to specify either x_dim or y_dim.

    :param im: The image: a 2D or 3D array.
    :param x_dim: An integer indicating the desired size, or None, to leave it loose.
    :param y_dim: An integer indicating the desired size, or None, to leave it loose.
    :return: A new image whose x_dim or y_dim matches the constraint
    """
    assert not (x_dim is None and y_dim is None), 'You can not leave both constraints at None!'

    x_dim = float('inf') if x_dim is None else x_dim
    y_dim = float('inf') if y_dim is None else y_dim

    box_aspect_ratio = x_dim/float(y_dim)
    image_aspect_ratio = im.shape[1] / float(im.shape[0])
    if image_aspect_ratio > box_aspect_ratio:  # Active constraint is width
        return imresize(im, size=(int(x_dim/image_aspect_ratio+.5), x_dim))
    else:  # Active constraint is height
        return imresize(im, size=(y_dim, int(y_dim*image_aspect_ratio+.5)))

def equalize_image_dims(list_of_images, x_dim = None, y_dim = None):
    """
    Resize images so that they match roughly in size although their aspect ratio will be preserved.
    :param list_of_images: A list of numpy arrays representing images (2D or 3D arrays)
    :param size: A 2-tuple specifying the desired (y_size, x_size).
        Each of (y_size, x_size) can be:
        - An integar, meaning that this axis of the image will remain equal or smaller than this number of pixels.
        - None, meaning that there is no constraint along this axis (e.g. (224, None) just states that the image will be
          scaled to 224 pixels in the vertical direction - the horizontal will be whatever size is needed to maintain
          the aspect ratio.
        - 'max': Meaning that we take the largest image size along this axis.
        - 'min': Meaning what we take the largest image size along this axis.

        The image will then be scaled so that the image size remains inside this box (although, unless the aspect ratio
        matches exactly, one dimension will be smaller).

    :return: Another list of images.
    """
    assert not (x_dim is None and y_dim is None), 'You can not leave both constraints at None!'
    if len(list_of_images)==0:
        return []
    x_dim = max(im.shape[1] for im in list_of_images) if x_dim=='max' else \
        min(im.shape[1] for im in list_of_images) if x_dim=='min' else \
        x_dim
    y_dim = max(im.shape[0] for im in list_of_images) if y_dim=='max' else \
        min(im.shape[0] for im in list_of_images) if y_dim=='min' else \
        y_dim
    new_list_of_images = [resize_while_preserving_aspect_ratio(im, x_dim=x_dim, y_dim=y_dim) for im in list_of_images]
    return new_list_of_images


from flask import Flask, redirect
from flask import url_for
import os
app = Flask(__name__, static_folder=r'C:\Users\matth\Documents\Projects\Family-Pic-Receiver\Test')

# Make the WSGI interface available at the top level so wfastcgi can get it.
wsgi_app = app.wsgi_app
fp=r'C:\Users\matth\Documents\Projects\Family-Pic-Receiver\Test'

class overallData:
    counter = 0

dataHold = overallData()

@app.route('/')
def displayImg():
    dataHold.counter += 1
    for counter,file in enumerate(os.listdir(fp)):
        if counter+1 == dataHold.counter:
            try:
                print(file)
                img_url = url_for('static',filename=file)
                return redirect(img_url)
            except:
                return '
Sad
'
    return '
Sorry
'

if __name__ == '__main__':
    app.run(port=8080,debug=True)


import logging
from collections import OrderedDict
import torch
import torch.nn as nn

from torch.nn.modules.activation import Sigmoid, ReLU

from allennlp.modules.conditional_random_field import ConditionalRandomField
from allennlp.modules import FeedForward
from allennlp.modules.span_extractors import SelfAttentiveSpanExtractor
from allennlp.modules.span_extractors import EndpointSpanExtractor
from allennlp.modules import TimeDistributed, Pruner
from allennlp.nn import util

from pytorch_models.utils import one_hot, get_activation_fn
from pytorch_models.utils import map_dict_builder
from pytorch_models.utils import create_mask, map_dict_builder
from pytorch_models.training import get_entity_loss

class SpanEmbedder(nn.Module):
    '''
    Create span embeddings

    Parameters
    ----------
    num_tags: label vocab size

    Returns
    -------
    arg_scores: tensor of scores (batch_size, trig_num, arg_num, 2)

    '''
    def __init__(self, input_dim, \
            use_endpoint = True,
            use_attention = True,
            combination = "x,y",
            num_width_embeddings = None,
            span_width_embedding_dim = None,
            span_end_is_exclusive = True):

        super(SpanEmbedder, self).__init__()

        self.input_dim = int(input_dim)
        self.use_endpoint = bool(use_endpoint)
        self.use_attention = bool(use_attention)
        self.combination = str(combination)

        if span_width_embedding_dim is None:
            self.num_width_embeddings = None
            feat_size = 0
        else:
            self.num_width_embeddings = num_width_embeddings
            feat_size = span_width_embedding_dim

        self.span_width_embedding_dim = span_width_embedding_dim
        self.span_end_is_exclusive = bool(span_end_is_exclusive)

        # Endpoint extractor
        if self.use_endpoint:
            self._endpoint_extractor = EndpointSpanExtractor( \
                input_dim = self.input_dim,
                combination = self.combination,
                num_width_embeddings = self.num_width_embeddings,
                span_width_embedding_dim = self.span_width_embedding_dim,
                bucket_widths = False)

        # Self-attentive span extractor
        if self.use_attention:
            self._attentive_extractor = \
               SelfAttentiveSpanExtractor(input_dim=self.input_dim)

        # Dimensionality of span/Input dimension for FFNN
        a = len(self.combination.split(','))*int(self.use_endpoint)
        b = int(self.use_attention)
        self.output_dim = self.input_dim*(a + b) + feat_size

        # Placeholder for decrement in span end index
        self.span_offset = None

    def forward(self, sequence_tensor, sequence_mask,
                      span_indices, span_mask):
        '''
        Parameters
        ----------
        sequence_tensor: sequence representation (batch_size, seq_len, embed_dim)
        sequence_mask: sequence mask (batch_size, seq_len)
        span_indices: tensor of span indices (batch_size, span_num, 2)
        span_mask: tensor of mask (batch_size, trig_num)

        Returns
        -------
        span_embed: tensor of span embeddings (batch_size, span_num, output_dim)
        '''

        # If span end indices are exclusive, subtract 1
        if self.span_end_is_exclusive:

            # Initialize span offset (decrement end indices)
            if (self.span_offset is None) or \
               (self.span_offset.shape != span_indices.shape):

                self.span_offset = torch.zeros_like(span_indices, \
                                                 requires_grad=False)
                self.span_offset[:,:,1] = - 1

            # Apply offset
            span_indices = span_indices + self.span_offset

        # Initialize output
        span_embed = []

        # Endpoint embedding
        if self.use_endpoint:
            span_embed.append(self._endpoint_extractor( \
                                    sequence_tensor = sequence_tensor,
                                    span_indices = span_indices,
                                    sequence_mask = sequence_mask,
                                    span_indices_mask = span_mask))

        # Attentive embedding
        if self.use_attention:
           span_embed.append(self._attentive_extractor( \
                                    sequence_tensor = sequence_tensor,
                                    span_indices = span_indices,
                                    sequence_mask = sequence_mask,
                                    span_indices_mask = span_mask))

        return torch.cat(span_embed, dim=2)

def span_labels_to_scores(labels, mask, num_tags, low_val, high_val, pruner):
    '''
    Use gold labels or CRF-predicted labels to generate label scores
    '''

    # Create binary, indicator labels
    binary_labels = (labels > 0).type(torch.FloatTensor)
    binary_labels = binary_labels.to(labels.device)

    # Number of elements to keep
    num_items_to_keep = max(int(binary_labels.sum(1).max()), 1)

    # Add dimension to binary labels so it can be treated as an embedding
    embeddings = binary_labels.unsqueeze(-1)

    # Get spans with positive labels
    _, mask_top, indices_top, prune_scores_top = \
                    pruner(embeddings, mask, num_items_to_keep)
    prune_scores_top = prune_scores_top.squeeze(-1)

    # Span scores
    label_scores = one_hot(labels, num_tags, \
                                         low_val = low_val,
                                         high_val = high_val)

    # Top span scores
    label_scores_top = util.batched_index_select( \
                                            target = label_scores,
                                            indices = indices_top)

    return (mask_top, indices_top, prune_scores_top, label_scores_top)

class PassThrough(nn.Module):
    '''
    Simple pass-through
    '''
    def __init__(self):
        super(PassThrough, self).__init__()

    def forward(self, x):
        return x

class SpanScorerGold(nn.Module):
    '''
    Span scorer using gold labels
    Convert labels to one hot encoding

    Parameters
    ----------
    num_tags: label vocab size

    Returns
    -------
    arg_scores: tensor of scores (batch_size, trig_num, arg_num, num_tags)

    '''
    def __init__(self, num_tags, low_val=-5, high_val=5):
        super(SpanScorerGold, self).__init__()

        self.num_tags = num_tags
        self.low_val = low_val
        self.high_val = high_val

        scorer = TimeDistributed(PassThrough())
        self._pruner = Pruner(scorer)

    def forward(self, labels, mask, embed):
        '''
        Parameters
        ----------
        span_labels: tensor of labels (batch_size, num_spans)

        Returns
        -------
        Pulled from AllenNLP GitHub:
            https://github.com/allenai/allennlp/blob/master/allennlp/modules/pruner.py

        embed_top : ``torch.FloatTensor``
            The representations of the top-k scoring items.
            Has shape (batch_size, num_items_to_keep, embedding_size).
        mask_top : ``torch.LongTensor``
            The corresponding mask for ``embed_top``.
            Has shape (batch_size, num_items_to_keep).
        indices_top : ``torch.IntTensor``
            The indices of the top-k scoring items into the original ``embeddings``
            tensor. This is returned because it can be useful to retain pointers to
            the original items, if each item is being scored by multiple distinct
            scorers, for instance. Has shape (batch_size, num_items_to_keep).
        top_item_scores : ``torch.FloatTensor``
            The values of the top-k scoring items.
            Has shape (batch_size, num_items_to_keep, 1).
        '''

        # Get scores from labels
        mask_top, indices_top, prune_scores_top, label_scores_top, \
             = span_labels_to_scores( \
                                        labels = labels,
                                        mask = mask,
                                        num_tags = self.num_tags,
                                        low_val = self.low_val,
                                        high_val = self.high_val,
                                        pruner = self._pruner)

        # Get top embeddings
        embed_top = util.batched_index_select(embed, indices_top)

        return (indices_top, embed_top, mask_top, prune_scores_top, label_scores_top)

class SpanScorerCRF(nn.Module):
    '''
    Span extractor
    '''
    def __init__(self, embed_size, label_map,
            low_val = -5,
            high_val = 5,
            constraints = None,
            incl_start_end = True,
            ):
        super(SpanScorerCRF, self).__init__()

        self.embed_size = embed_size
        self.label_map = label_map
        self.low_val = low_val
        self.high_val = high_val
        self.constraints = constraints
        self.incl_start_end = incl_start_end

        # Number of positive tags
        self.num_pos_tags = len(label_map[1:])

        # label_map to/from BIO (0th label is negative)
        self.mapping_BIO = label_map[:] + label_map[1:]

        # Dicts for mapping to/from label indices
        self.label_to_id, self.id_to_label = \
                                      map_dict_builder(self.mapping_BIO)

        # Number of tags, including BI prefixes
        self.num_tags = len(self.mapping_BIO)

        # Linear projection layer
        self.projection = nn.Linear(embed_size, self.num_tags)

        # Create event-specific CRF
        self.crf = ConditionalRandomField( \
                        num_tags = self.num_tags,
                        constraints = constraints,
                        include_start_end_transitions = incl_start_end)

        # Dummy pruner
        scorer = TimeDistributed(PassThrough())
        self._pruner = Pruner(scorer)

    def forward(self, seq_tensor, seq_mask, span_map, span_mask, embed, spans, \
                                                    span_labels=None):
        '''
        Generate predictions
        '''
        # Dimensionality
        batch_size, max_seq_len, input_dim = tuple(seq_tensor.shape)

        # Project input tensor sequence to logits
        logits = self.projection(seq_tensor)

        # Best path
        best_paths = self.crf.viterbi_tags( \
                                        logits = logits,
                                        mask = seq_mask)

        # Separate predictions and score
        seq_pred, score = zip(*best_paths)
        seq_pred = list(seq_pred)

        '''
        Process predictions
        '''
        # Get spans from sequence tags
        #   Converts list of list of predicted label indices to
        #   tensor of size (batch_size, num_spans)
        span_pred = self._seq_tags_to_spans(seq_pred, span_map)

        # Get scores from labels
        mask_top, indices_top, prune_scores_top, label_scores_top, \
             = span_labels_to_scores( \
                                        labels = span_pred,
                                        mask = span_mask,
                                        num_tags = self.num_tags,
                                        low_val = self.low_val,
                                        high_val = self.high_val,
                                        pruner = self._pruner)

        # Get top embeddings
        embed_top = util.batched_index_select(embed, indices_top)

        '''
        Calculate loss
        '''
        # No labels provided
        if span_labels is None:
            loss = None

        # Span labels provided (i.e. training)
        else:
            # Convert span representation to sequence tags
            seq_true = self._spans_to_seq_tags(span_labels, spans,
                                                            max_seq_len)
            # Get loss (negative log likely)
            loss = -self.crf( \
                                inputs = logits,
                                tags = seq_true,
                                mask = seq_mask)

        return (embed_top, mask_top, indices_top, prune_scores_top,
                                   label_scores_top, loss)

    def _to_BIO(self, span_label, start, end):
        '''
        Convert span representation to BIO representation

        Parameters
        ----------
        span_label: scaler tensor, span label index
        start: scaler tensor, span start index
        end: scaler tensor, span end index

        '''

        # Span length
        n = end - start

        # Inside label
        I = span_label + self.num_pos_tags

        # Initialize to inside label
        seq_labels = torch.zeros(n).type(torch.LongTensor)
        seq_labels.fill_(I)

        # Begin label
        seq_labels[0] = span_label

        #REMOVE
        #seq_labels.fill_(span_label)

        return seq_labels

    def _from_BIO(self, seq_label):
        '''
        Convert BIO representation to span representation

        Parameters
        ----------
        seq_label: int, span label with BIO indices
        '''

        logging.warn("="*300)
        logging.warn("TODO replace this fucntion with pytorch_models.crf.tag_to_span_lab ")
        logging.warn("="*300)

        is_outside = 0
        is_begin = 0
        is_inside = 0

        # Convert sequence label to span label (i.e. convert BI)
        if seq_label > self.num_pos_tags:
            span_label = seq_label - self.num_pos_tags
            is_inside = 1
        elif seq_label > 0:
            span_label = seq_label
            is_begin = 1
        else:
            span_label = seq_label
            is_outside = 1

        return (span_label, is_outside, is_begin, is_inside)

    def _seq_tags_to_spans(self, seq_tags, span_map):
        '''
        Convert sequence tags to span labels

        Parameters
        ----------
        seq_tags: list of list of label indices
                 i.e. list of sentences, where each sentence
                      is a list of label indices

        Returns
        -------
        span_labels: tensor of shape (batch_size, num_spans)

        '''

        # Get inputs for tensor initialization
        batch_size = len(seq_tags)
        num_spans = len(span_map)

        # Initialize span labels to null label
        span_labels = torch.zeros(batch_size, num_spans).type( \
                                                       torch.LongTensor)

        # Loop on sequences
        for i_seq, seq in enumerate(seq_tags):

            # Loop on spans
            for lab, start, end in self._BIO_to_span(seq):

                # Token indices of current span
                idx = (start, end)

                # Span in map
                if idx in span_map:

                    # Span index within tensor
                    i_span = span_map[idx]

                    # Update label tensor
                    span_labels[i_seq, i_span] = lab

                # Span not in map
                else:
                    pass
                    logging.warn("span not in map:\t{}".format(idx))

        return span_labels

    def _spans_to_seq_tags(self, labels, spans, max_seq_len):
        '''
        Convert span labels to sequence labels

        Parameters
        ----------
        labels: tensor of label indices (batch_size, num_spans)
        spans: tensor of span indices (batch_size, num_spans, 2)
        max_seq_len: int, maximum sequence length
        num_pos_tags: int, number of non-negative labels

        Returns
        -------
        seq_tags: tensor with BIO label indices (batch_size, max_seq_len)
        '''

        # Label dimensionality
        batch_size, num_spans = tuple(labels.shape)

        # Initialize sequence tags to 0
        # (batch_size, max_seq_len)
        seq_tags = torch.zeros(batch_size, max_seq_len).type( \
                                                       torch.LongTensor)

        # Find indices of all nonnegative (0) labels
        # (n, d)
        #   n = number of nonzero elements,
        #   d = number of dimensions in labels (i.e. 2)
        non_neg_indices = torch.nonzero(labels)

        # Iterate over nonnegative labels
        for indices in non_neg_indices:

            # Sequence and span indices
            # indices_shape (2)
            i_seq = indices[0]
            i_span = indices[1]

            # Label
            # (1)
            lab = labels[i_seq, i_span]

            # Span start and stop indices
            # (2)
            span = spans[i_seq, i_span]
            start = span[0]
            end = span[1]

            # Update sequence tags with BI labels
            seq_tags[i_seq, start:end] = self._to_BIO(lab, start, end)

        return seq_tags

    def _BIO_to_span(self, seq):
        '''

        Finds spans in BIO sequence

        NOTE: start span index is inclusive, end span index is exclusive
                e.g. like Python lists

        '''

        logging.warn("="*300)
        logging.warn("TODO replace this fucntion with pytorch_models.crf.BIO_to_span")
        logging.warn("="*300)

        spans = []
        begin_count = 0
        start = -1
        end = -1
        active_tag = None

        if not any(seq):
            return []

        for i_tok, x in enumerate(seq):

            # Convert current sequence tag label to span label
            tag, is_outside, is_begin, is_inside = self._from_BIO(x)

            # Outside label
            if is_outside:

                # The span has ended
                if active_tag is not None:
                    spans.append((active_tag, start, end))

                # Not in a span
                active_tag = None

            # Span beginning
            elif is_begin:

                # The span has ended
                if active_tag is not None:
                    spans.append((active_tag, start, end))

                # Update active tag
                active_tag = tag

                # Index of current span start
                start = i_tok
                end = i_tok + 1

                # Increment begin count
                begin_count += 1

            # Span inside and current tag matches active tag
            # e.g. well-formed span
            elif is_inside and (tag == active_tag):
                end += 1

            # Ill formed span
            elif is_inside and (tag != active_tag):

                # Capture end of valid span
                if active_tag is not None:
                    spans.append((active_tag, start, end))

                # Not in a span
                active_tag = None

            else:
                raise ValueError("could not assign label")

        # Last token might be part of a valid span
        if active_tag is not None:
            spans.append((active_tag, start, end))

        # Get span count
        span_count = len(spans)

        if True and (begin_count != span_count):
            msg = \
            '''Count mismatch:
            seq = {}
            Begin count = {}
            span count = {}'''.format(seq, begin_count, span_count)
            logging.warn(msg)

        return spans

class BeamScorer(nn.Module):
    '''

    '''
    def __init__(self, scorer, agg_type):
        super(BeamScorer, self).__init__()
        self._scorer = scorer
        self.agg_type = agg_type

    def forward(self, candidates):

        # Apply scorer
        # [batch_size, n_spans, n_labels]
        scores = self._scorer(candidates)

        # Get overall score for non-null (positive) labels
        # Null labels represented by label index 0
        if self.agg_type == 'max':
            pos_scores, _ = scores[:,:,1:].max(dim=-1)
        elif self.agg_type == 'sum':
            pos_scores = scores[:,:,1:].sum(dim=-1)
        else:
            raise ValueError("incorrect agg_type: {}".format(self.agg_type))

        # Add last dimension for compliance with Pruner API
        # [batch_size, n_spans, 1]
        return pos_scores.unsqueeze(-1)

class SpanScorerPrune(nn.Module):
    '''
    Span scorer

    Parameters
    ----------
    num_tags: label vocab size

    Returns
    -------
    arg_scores: tensor of scores (batch_size, trig_num, arg_num, num_tags)

    '''
    def __init__(self, input_dim, hidden_dim, num_tags, \
            activation = 'relu',
            dropout = 0.0,
            spans_per_word = 2,
            agg_type = 'max',
            loss_reduction = 'sum'):
        super(SpanScorerPrune, self).__init__()

        self.input_dim = input_dim
        self.hidden_dim = hidden_dim
        self.num_tags = num_tags

        self.activation = activation
        self.activation_fn = get_activation_fn(activation)

        self.dropout = dropout
        self.spans_per_word = spans_per_word
        self.agg_type = agg_type
        self.loss_reduction = loss_reduction

        self.num_layers = 1
        self.neg = -1e20

        '''
        Create classifier
        '''
        # Feedforward neural network for predicting span labels
        self._label_ffnn = FeedForward( \
                    input_dim = self.input_dim,
                    num_layers = self.num_layers,
                    hidden_dims = self.hidden_dim,
                    activations = self.activation_fn,
                    dropout = self.dropout)

        # Span classifier
        self._label_scorer = torch.nn.Sequential(
            TimeDistributed(self._label_ffnn),
            TimeDistributed(torch.nn.Linear(self.hidden_dim, self.num_tags)))

        '''
        Create pruner
        '''
        # Feedforward neural network for pruning score
        #self._pruning_ffnn = FeedForward( \
        #            input_dim = self.input_dim,
        #            num_layers = self.num_layers,
        #            hidden_dims = self.hidden_dim,
        #            activations = self.activation,
        #            dropout = self.dropout)

        # Pruning scoring function
        #self._pruning_scorer = torch.nn.Sequential(
        #    TimeDistributed(self._pruning_ffnn),
        #    TimeDistributed(torch.nn.Linear(self.hidden_dim, 1)))

        # Pruner
        self._pruning_scorer = BeamScorer(self._label_scorer, self.agg_type)
        self._pruner = Pruner(self._pruning_scorer)

    def forward(self, embed, mask, seq_lengths, span_labels=None):
        '''
        Parameters
        ----------
        span_labels: tensor of labels (batch_size, num_spans)

        Returns
        -------
        Pulled from AllenNLP GitHub:
            https://github.com/allenai/allennlp/blob/master/allennlp/modules/pruner.py

        embed_top : ``torch.FloatTensor``
            The representations of the top-k scoring items.
            Has shape (batch_size, num_items_to_keep, embedding_size).
        mask_top : ``torch.LongTensor``
            The corresponding mask for ``embed_top``.
            Has shape (batch_size, num_items_to_keep).
        indices_top : ``torch.IntTensor``
            The indices of the top-k scoring items into the original ``embeddings``
            tensor. This is returned because it can be useful to retain pointers to
            the original items, if each item is being scored by multiple distinct
            scorers, for instance. Has shape (batch_size, num_items_to_keep).
        top_item_scores : ``torch.FloatTensor``
            The values of the top-k scoring items.
            Has shape (batch_size, num_items_to_keep, 1).
        '''

        # Number of spans to keep by sentence
        # (batch_size)
        num_to_keep = seq_lengths*self.spans_per_word
        num_to_keep = torch.max(num_to_keep, torch.ones_like(num_to_keep))

        # Apply pruner to embeddings
        # embed_top (batch_size, max_num_to_keep, embed_dim)
        # mask_top, indices_top, prune_scores_top (batch_size, max_num_to_keep)
        # prune_scores_top (batch_size, max_num_to_keep, 1)
        embed_top, mask_top, indices_top, prune_scores_top = \
                            self._pruner(embed, mask, num_to_keep)
        prune_scores_top = prune_scores_top.squeeze(-1)

        # Compute label scores
        # (batch_size, num_spans, num_tags)
        label_scores = self._label_scorer(embed)

        # Give large negative scores to the masked-out values.
        label_scores = util.replace_masked_values( \
                                                tensor = label_scores,
                                                mask = mask.unsqueeze(-1),
                                                replace_with = self.neg)

        # Top span scores
        label_scores_top = \
                util.batched_index_select(label_scores, indices_top)

        if span_labels is None:
            loss = None
        else:
            loss = get_entity_loss( \
                    scores = label_scores,
                    labels = span_labels,
                    mask = mask,
                    reduction = self.loss_reduction)

        return (embed_top, mask_top, indices_top, prune_scores_top,
                                         label_scores_top, loss)


#!/usr/bin/env python3

import argparse
import os

import command_add
import command_show
import command_update

def create_parser():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(help="subcommands")

    parser.add_argument("-i", "--inventory", default="data/inventory.db", help="path to inventory database")
    parser.add_argument("-m", "--market", default="data/market.json", help="path to market database")

    parser_add = subparsers.add_parser("add", help="add an item to inventory")
    parser_add.set_defaults(command=command_add.run)
    parser_add.add_argument("item", nargs="?")
    parser_add.add_argument("count", nargs="?", default=1, type=int)

    parser_show = subparsers.add_parser("show", help="show full inventory")
    parser_show.set_defaults(command=command_show.run)
    parser_show.add_argument("-d", "--ducats", action="store_true", help="sort by ducats/price value")
    parser_show.add_argument("-p", "--plats", action="store_true", help="sort by price")

    parser_update = subparsers.add_parser("update")
    parser_update.set_defaults(command=command_update.run)

    return parser

if __name__ == "__main__":
    parser = create_parser()
    args = parser.parse_args()
    os.makedirs("data", exist_ok=True)
    if "command" in args:
        args.command(args)
    else:
        parser.print_help()


"""
Copyright 2020 The OneFlow Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import unittest
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import tensorflow_addons as tfa
import test_global_storage
from test_util import GenArgList

import oneflow.compatible.single_client.unittest
from oneflow.compatible import single_client as flow

gpus = tf.config.experimental.list_physical_devices("GPU")
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

def compare_with_tensorflow_addons_lamb(
    test_case,
    device_type,
    x_shape,
    beta1,
    beta2,
    epsilon,
    weight_decay,
    learning_rate,
    train_iters,
):
    assert device_type in ["gpu", "cpu"]
    flow.clear_default_session()
    func_config = flow.FunctionConfig()
    func_config.default_data_type(flow.float32)

    @flow.global_function(type="train", function_config=flow.FunctionConfig())
    def testLAMB(
        random_mask: flow.typing.Numpy.Placeholder(x_shape, dtype=flow.float32)
    ) -> flow.typing.Numpy:
        with flow.scope.placement(device_type, "0:0-0"):
            x = flow.get_variable(
                name="x",
                shape=x_shape,
                dtype=flow.float32,
                initializer=flow.random_uniform_initializer(minval=-10, maxval=10),
                trainable=True,
            )
            loss = flow.math.reduce_mean(x + random_mask)
            flow.optimizer.LAMB(
                flow.optimizer.PiecewiseConstantScheduler([], [learning_rate]),
                beta1=beta1,
                beta2=beta2,
                epsilon=epsilon,
                weight_decay=weight_decay,
            ).minimize(loss)
            return x

    random_masks_seq = []
    for i in range(train_iters + 1):
        random_masks_seq.append(np.random.uniform(size=x_shape).astype(np.float32))
    x_list = []
    init_value = None
    for i in range(train_iters + 1):
        x = testLAMB(random_masks_seq[i])
        x_list.append(x)
        if i == 0:
            init_value = np.copy(x)
    var = tf.Variable(init_value)
    opt = tfa.optimizers.LAMB(
        learning_rate=learning_rate,
        beta_1=beta1,
        beta_2=beta2,
        epsilon=epsilon,
        weight_decay_rate=weight_decay,
    )
    var_list = []
    for i in range(train_iters):
        with tf.GradientTape() as tape:
            if i == 0:
                var0 = tf.identity(var)
                var_list.append(var0)
            random_mask = tf.Variable(random_masks_seq[i])
            loss = tf.reduce_mean(var + random_mask)
        gradients = tape.gradient(loss, var)
        opt.apply_gradients(zip([gradients], [var]))
        var_list.append(var.numpy())
    case = (
        device_type,
        x_shape,
        beta1,
        beta2,
        epsilon,
        weight_decay,
        learning_rate,
        train_iters,
    )
    test_case.assertTrue(len(x_list) == len(var_list))
    for (i, o, t) in zip(range(len(var_list)), x_list, var_list):
        diff = o - t
        test_case.assertTrue(
            np.allclose(x_list[i], var_list[i], rtol=0.001, atol=0.001), (i, case, diff)
        )
    diff = x.flatten() - var.numpy().flatten()
    test_case.assertTrue(
        np.allclose(x.flatten(), var.numpy().flatten(), rtol=0.001, atol=0.001),
        (case, diff),
    )

@flow.unittest.skip_unless_1n1d()
class TestLamb(flow.unittest.TestCase):
    def test_lamb(test_case):
        arg_dict = OrderedDict()
        arg_dict["test_case"] = [test_case]
        arg_dict["device_type"] = ["cpu", "gpu"]
        arg_dict["x_shape"] = [(10,)]
        arg_dict["beta1"] = [0.9]
        arg_dict["beta2"] = [0.999]
        arg_dict["epsilon"] = [1e-06]
        arg_dict["weight_decay"] = [0.01]
        arg_dict["learning_rate"] = [0.0001]
        arg_dict["train_iters"] = [10]
        for arg in GenArgList(arg_dict):
            compare_with_tensorflow_addons_lamb(*arg)

if __name__ == "__main__":
    unittest.main()


import os
from setuptools import setup

with open(os.path.join(os.path.dirname(__file__), "README.md"), encoding="utf-8") as f:
    long_description = f.read()

module_version = "0.1.5"

setup(
    name="DailyLogger",
    packages=["DailyLogger"],
    version=module_version,
    license="MIT",
    description="A basic daily logger to log python projects.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    author="Samyak Ratna Tamrakar",
    author_email="samyak.r.tamrakar@gmail.com",
    url="https://github.com/srtamrakar/python-logger",
    download_url=f"https://github.com/srtamrakar/python-logger/archive/v_{module_version}.tar.gz",
    keywords=["log", "logger", "logging"],
    install_requires=["multiprocessing_logging==0.3.0"],
    classifiers=[
        "Development Status :: 4 - Beta",  # Either"3 - Alpha", "4 - Beta" or "5 - Production/Stable"
        "Intended Audience :: Developers",  # Define that your audience are developers
        "Topic :: Software Development :: Build Tools",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.4",
        "Programming Language :: Python :: 3.5",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
    ],
)


import datetime
from typing import Dict, Iterable, Mapping

import attr

import aiopg.sa
from aiopg.sa.result import ResultProxy, RowProxy
import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import Insert
from sqlalchemy.sql.selectable import Select
from sqlalchemy.sql import and_, not_
from sqlalchemy.sql.schema import Column

DEFAULT_PAGE_SIZE = 100

class BasePostgresClient:
    def __init__(
        self,
        usecase_class,
        engine: aiopg.sa.Engine,
        table: sa.Table,
        db_generated_fields: Iterable[str] = None,
    ):
        self.usecase_class = usecase_class
        self.engine = engine
        self.table = table
        self.db_generated_fields = db_generated_fields or ["created_at", "updated_at"]

    async def insert(self, usecase):
        serialized_usecase: Dict = self._serialize_for_db(usecase)
        async with self.engine.acquire() as conn:
            statement: Insert = (
                self.table.insert()
                .values(**serialized_usecase)
                .returning(*[column for column in self.table.columns])
            )
            results: ResultProxy = await conn.execute(statement)
            return await results.fetchone()

    async def select_first_where(
        self, include: Mapping = None, exclude: Mapping = None
    ):
        results = await self.select_where(include=include, exclude=exclude, page_size=1)
        if results:
            return results[0]
        return None

    async def select_where(
        self, include: Mapping = None, exclude: Mapping = None, page=0, page_size=None
    ):
        where_clause = self._generate_where_clause(include, exclude)
        page_size = page_size if page_size else DEFAULT_PAGE_SIZE
        async with self.engine.acquire() as conn:
            statement: Select = self.table.select().where(where_clause)
            paginated_statement = self._paginate_query(statement, page, page_size)
            results: ResultProxy = await conn.execute(paginated_statement)
            return [await self._deserialize_from_db(result) async for result in results]

    async def update_where(
        self, set_values: Mapping, include: Mapping = None, exclude: Mapping = None
    ):
        where_clause = self._generate_where_clause(include, exclude)
        async with self.engine.acquire() as conn:
            statement = self.table.update.where(where_clause)

    def _generate_where_clause(self, include: Mapping = None, exclude: Mapping = None):
        """Turn inclusion/exclusion maps into SQLAlchemy `where` clause"""
        inclusion_ands = []
        exclusion_ands = []
        if include:
            for field, includes in include.items():
                table_col: Column = getattr(self.table.c, field)
                if _isiterable(includes):
                    # Use SQL [column] IN [(values)]
                    inclusion_ands.append(table_col.in_(includes))
                else:
                    # Use SQL [column] = [value]
                    inclusion_ands.append(table_col == includes)
        if exclude:
            for field, excludes in exclude.items():
                table_col: Column = getattr(self.table.c, field)
                if _isiterable(excludes):
                    # Use SQL [column] NOT IN [(values)]
                    exclusion_ands.append(not_(table_col.in_(excludes)))
                else:
                    # Use SQL [column] != [value]
                    exclusion_ands.append(table_col != excludes)
        return and_(*inclusion_ands, *exclusion_ands)

    def _generate_values_clause(self, set_values: Mapping):
        pass

    def _paginate_query(self, where_clause, page=0, page_size=None):
        if page_size:
            where_clause = where_clause.limit(page_size)
        if page:
            where_clause = where_clause.offset(page * page_size)
        return where_clause

    async def _deserialize_from_db(self, row: RowProxy):
        # returns attrs object if successful
        row_dict = dict(row)
        return self.usecase_class(**row_dict)

    def _serialize_for_db(self, usecase) -> Dict:
        # at this point we're assuming attrs objects for usecases
        usecase_dict: Dict = attr.asdict(usecase)
        for db_generated_field in self.db_generated_fields:
            if usecase_dict.get(db_generated_field) is None:
                # inserting a non-nullable field with value None will result in a
                # `psycopg2.IntegrityError: null value in column violates not-null constraint`
                # we delete the value from the dict instead
                del usecase_dict[db_generated_field]
        for k, v in usecase_dict.items():
            if isinstance(v, datetime.datetime):
                usecase_dict[k]: str = v.isoformat()
        return usecase_dict

def _isiterable(var) -> bool:
    return isinstance(var, Iterable) and not isinstance(var, str)