import torch
import torch_geometric
import networkx as nx
import random

TorchNodes = torch.LongTensor
TorchEdges = torch.LongTensor
TorchMappings = torch.LongTensor
TorchEdgeMask = torch.BoolTensor

def augment(s_1: TorchEdges, s_1_nodes: TorchNodes, s_2: TorchEdges,  s_2_nodes: TorchNodes, G_noisy: nx.Graph, t: set(("src", "tar"))) -> (TorchEdges, TorchEdges, list(("src", "tar")), list(("src", "tar"))):
    d1 = torch_geometric.data.Data(edge_index=s_1)
    d1.num_nodes = 0
    d2 = torch_geometric.data.Data(edge_index=s_2)
    d2.num_nodes = 0
    s_1_nx = torch_geometric.utils.to_networkx(d1, to_undirected=False)
    s_2_nx = torch_geometric.utils.to_networkx(d2, to_undirected=False)
    s_12_nx = G_noisy.subgraph(list(s_1_nx.nodes) + list(s_2_nx.nodes)).copy()

    y_s_12 = list(filter(lambda x: s_12_nx.has_edge(x[0], x[1]), t))

    y_s_12_sample = []
    if len(y_s_12) != 0:
        y_s_12_sample = random.sample(y_s_12, random.randint(
            0,  max(1, min(len(y_s_12) - 1, min((s_1[0].size(0) // 2) - 1, (s_2[0].size(0) // 2) - 1)))))

    if (len(s_1_nx.edges)) > 2:
        s_1_nx.remove_edges_from(y_s_12_sample)
        s_1_nx.remove_edges_from(list(map(swap, y_s_12_sample)))
    if (len(s_2_nx.edges)) > 2:
        s_2_nx.remove_edges_from(y_s_12_sample)
        s_2_nx.remove_edges_from(list(map(swap, y_s_12_sample)))

    y_neg = [(random.choice(list(s_1_nx.nodes) + list(s_2_nx.nodes)), random.randint(
        len(G_noisy.nodes) + 1, len(G_noisy.nodes) * 2) - 1) for _ in range(len(y_s_12_sample))]

    y_neg_s_1 = list(filter(lambda x: x[0] in s_1_nx.nodes, y_neg))
    y_neg_s_2 = list(filter(lambda x: x[0] in s_2_nx.nodes, y_neg))
    s_1_nx.add_edges_from(y_neg_s_1)
    s_1_nx.add_edges_from(list(map(swap, y_neg_s_1)))
    s_2_nx.add_edges_from(y_neg_s_2)
    s_2_nx.add_edges_from(list(map(swap, y_neg_s_2)))

    return torch.LongTensor(list(s_1_nx.edges)).t().contiguous(), torch.LongTensor(list(s_2_nx.edges)).t().contiguous(), y_s_12, y_s_12_sample, y_neg

def swap(x):
    return (x[1], x[0])


import pygame as pg
import random
from utils import LIST_DICTIONARY, COLOR_LINES, ORANGE, PURPLE, FONT

class GuessWord:
    def __init__(self):
        self.guess_word = ""
        self.x = 150
        self.w = 190
        self.new_x = 40
        self.new_y = 50
        self.space = 10
        self.y, self.h, = 130, 130
        self.color = COLOR_LINES
        self.rect = []
        self.txt_surface = FONT.render('', True, self.color)
        self.txt_surf_win = FONT.render('You won!', True, ORANGE)
        self.rect_win = pg.Rect(self.x + 150, self.y + 60, self.w - 50, self.h - 80)

    def random_word(self):
        rand_num = random.randint(0, len(LIST_DICTIONARY) - 1)
        self.guess_word = LIST_DICTIONARY[rand_num]
        return self.guess_word

    def check_in(self, letter, print_word):
        for i in range(len(self.guess_word)):
            if self.guess_word[i] == letter:
                print_word[i] = letter
        return print_word

    def draw_lines(self, screen, print_word):
        x = self.x
        w = self.w
        new_x = self.new_x
        space = self.space
        y, h = self.y, self.h
        txt = [" " if not i else i for i in print_word]
        for i in range(len(self.guess_word)):
            self.rect.append(pg.Rect(x, y-self.new_y, new_x, new_x))
            self.txt_surface = FONT.render(txt[i], True, self.color)
            screen.blit(self.txt_surface, (self.rect[i].x + 5, self.rect[i].y-5))
            pg.draw.line(
                screen, COLOR_LINES, (x, y), (w, h), 5)
            pg.draw.rect(screen, PURPLE, self.rect[i], 2)
            x = x + new_x + space
            w = w + new_x + space
        if all([' ' != i for i in txt]):
            screen.blit(self.txt_surf_win, (self.rect_win.x + 5, self.rect_win.y - 5))
            pg.draw.rect(screen, PURPLE, self.rect_win, 2)


from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'index', views.index, name='index'),
    url(r'create', views.create_role, name='create'),
    url(r'update', views.update_role, name='update'),
    url(r'delete', views.delete_role, name='delete'),
    url(r'assign', views.assign_role, name='assign'),
    url(r'deassign', views.deassign_role, name='deassign'),
]


# -*- coding: utf-8 -*-
import os, sys, cgi, time, types, exif

PATH     = u"/var/www/public/fotos"
URL      = u"https://example.com/fotos"
TEMPLATE = open(os.path.join(os.path.dirname(__file__), "template.html")).read().decode("utf8")
NPERPAGE = 10

print >> sys.stderr, "booted"

listing, ts = None, 0

def application(env, start_resp):
    global listing, ts

    def ret_badrequest():
        text = "400 Bad Request"
        start_resp("400 Bad Request",
                [("Content-Type", "text/plain"),
                 ("Content-Length", str(len(text)))])
        return [text]

    def ret_response(ctype, data):
        ctype = str(ctype)
        if not type(data) == types.StringType:
            data = data.encode("utf8")
        start_resp("200 OK", [
            ("Cache-Control", "max-age=0, no-store"),
            ("Content-Type", ctype),
            ("Content-Length", str(len(data)))])
        return [data]

    cts = os.path.getmtime(PATH) #int(time.time())
    #if cts - ts > 1200:
    if not cts == ts:
        ts = cts
        print >> sys.stderr, "reload listing"
        listing = []
        for root, dirs, files in os.walk(PATH):
            for f in files:
                if not f.startswith(".") and os.path.splitext(f)[1].lower() in [".jpg", ".jpeg"]:
                    path = os.path.join(root, f)
                    created = exif.parse(path).get("DateTimeOriginal")
                    url = u"%s%s" % (URL, path[len(PATH):])
                    listing.append((created, url))
        listing = sorted(listing)

    env_script = env["SCRIPT_URI"]

    params = env["QUERY_STRING"]
    params = cgi.parse_qs(params)
    params = dict([(key, params[key][0].strip()) for key in params])
    env_offs  = params.get("offs",  len(listing) - 1)
    env_count = params.get("count", NPERPAGE)
    del params
    try:
        env_offs  = int(env_offs)
        env_count = int(env_count)
    except ValueError:
        return ret_badrequest()
    if env_offs < 0:
        #env_offs = len(listing) - 1
        return ret_badrequest()
    if env_count <= 0:
        #env_count = NPERPAGE
        return ret_badrequest()

    count = env_count
    if env_offs - count + 1 < 0:
        count = env_offs + 1

    page = listing[env_offs - count + 1: env_offs + 1]
    page.reverse()
    page = [u'
\n
\n%s\n
' % (url, url, created) for created, url in page]
    page = "\n".join(page)

    linkprev, linknext = u"prev", u"next"

    if env_offs - count >= 0:
        linkprev = u'prev' % (
            env_script, env_offs - env_count, env_count)

    if env_offs < len(listing) - 1:
        linknext = u'next' % (
            env_script, env_offs + env_count, env_count)

    navi = u'\n%s\n%s\n' % (
        linkprev, linknext)

    content = u'
\n%s
\n%s
\n%s
' % (navi, page, navi)
    content = TEMPLATE % content

    return ret_response("text/html", content)


from config import Configuration
from embedder import train_embedder
import os
import pandas as pd

data_name = 'CarEvaluation'
print(data_name)

# data = pd.read_csv('data/{}.csv'.format(data_name)).drop('class',axis=1)
data = pd.read_csv('data/{}.csv'.format(data_name))

embedding_size = 4
corruption_ratio_train = 0.0
corruption_ratio_validation = 0.05
config_data = Configuration(train_batch_size=128, val_batch_size= int(len(data) * 0.2),
                            embedding_size=embedding_size, max_iteration=20000,
                            print_loss_every=100,
                            LOG_DIR = './{}/embedding_{}/corruption_{}/validation_{}/class_var/log/'.format(
                                data_name, embedding_size, corruption_ratio_train,
                                corruption_ratio_validation),
                            corruption_ratio_train=corruption_ratio_train,
                            corruption_ratio_validation = corruption_ratio_validation,
                            model_save_filename = 'model.ckpt',
                            metadata_filename = 'metadata.tsv',
                            )

if not os.path.exists(config_data.LOG_DIR):
    os.makedirs(config_data.LOG_DIR)

with open(os.path.join(config_data.LOG_DIR, 'config.txt'), 'w') as f:
    f.write(repr(config_data))

for col in data.columns:
    if data[col].dtype == int or data[col].dtype == float:
        data[col] = data[col].astype('str')

train_embedder(data, config_data)


import sys
from flask import Flask, jsonify, make_response, request, abort

app = Flask(__name__)

@app.after_request
def after_request(data):
	response = make_response(data)
	response.headers['Content-Type'] = 'application/json'
	response.headers['Access-Control-Allow-Origin'] = '*'
	response.headers['Access-Control-Allow-Headers'] = "Origin, X-Requested-With, Content-Type, Accept"
	response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, DELETE'
	return response

tasks = []

@app.errorhandler(400)
def bad_request(error):
    return make_response(jsonify( { 'error': 'Bad Request' } ), 400)

@app.errorhandler(404)
def not_found(error):
    return make_response(jsonify( { 'error': 'Not Found' } ), 404)

@app.route('/tasks', methods = ['POST'])
def create_task():
    if not request.json or not 'task' in request.json:
        abort(400)
    task = {
        'id': len(tasks),
        'task': request.json['task']
    }
    tasks.append(task)
    return jsonify( { 'task': task } ), 201

@app.route('/tasks', methods = ['GET'])
def get_tasks():
    return jsonify( { 'tasks': tasks } )

@app.route('/tasks/<int:task_id>', methods = ['GET'])
def get_task(task_id):
    task = filter(lambda t: t['id'] == task_id, tasks)
    if len(task) == 0:
        abort(404)
    return jsonify( { 'task': task[0] } )

@app.route('/tasks/<int:task_id>', methods = ['PUT'])
def update_task(task_id):
    task = filter(lambda t: t['id'] == task_id, tasks)
    if len(task) == 0 or not request.json:
        abort(404)
    if 'task' in request.json and type(request.json['task']) is not unicode:
        abort(400)
    task[0]['task'] = request.json.get('task', task[0]['task'])
    return jsonify( { 'task': task[0] } )

@app.route('/tasks/<int:task_id>', methods = ['DELETE'])
def delete_task(task_id):
    task = filter(lambda t: t['id'] == task_id, tasks)
    if len(task) == 0:
        abort(404)
    tasks.remove(task[0])
    return jsonify( { 'result': True } )

if __name__ == '__main__':
    app.run(debug = True)


import copy
import os, time
import augment
import pretrained_model

import albumentations as A
from albumentations.pytorch import ToTensorV2
import cv2
import torch
import torch.backends.cudnn as cudnn
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.utils.data import Dataset, DataLoader
import torchvision.models as models
from tqdm import tqdm
from time import sleep

cudnn.benchmark = True

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
    since = time.time() # get the starting time of training

    best_loss = 100000.0
    tmax_factor = 1.5 # with warm restart, multiply the max factor by this amount
    tmax = 10 # after tmax iterations, the learning rate is reset

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            iterations = len(dataloaders[phase])

            # Iterate over data.
            pbar = tqdm(total=iterations,desc=phase,ncols=70)
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                output_tensor = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # Run the forward pass and track history if only in training
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    preds = outputs
                    loss = criterion(outputs, output_tensor)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                pbar.update(1)
                sleep(0.01) #delay to print stats
            pbar.close()

            if phase == 'train': #adjust the learning rate if training
                scheduler.step()

            # With the cosine annealing scheduler, the learning rate is
            # gradually reduced.  If the learning rate is very small, then
            # reset the scheduler to do a warm restart
            if optimizer.param_groups[0]['lr'] < .0000001:
                tmax = int(tmax * tmax_factor)
                scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=tmax, verbose=True)
                print ("New T_max: " + str(tmax))

            epoch_loss = running_loss / dataset_sizes[phase]

            print('{} Loss: {:.4f}'.format(phase, epoch_loss))

            # deep copy the model
            if phase == 'val' and epoch_loss < best_loss:
                best_loss = epoch_loss
                best_model_wts = copy.deepcopy(model.state_dict())
                print ("Saving new best model...")
                torch.save(model, 'test.pt')

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_loss))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model

if __name__ == '__main__':
    train_fp, val_fp = augment.setup_dir()
    train_d, val_d = augment.create_datasets(train_fp, val_fp)

    image_datasets = {}
    image_datasets['train'] = train_d
    image_datasets['val'] = val_d

    dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], \
                   batch_size=32, shuffle=True, num_workers=12) for x in ['train', 'val']}

    dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}
    print (dataset_sizes)

    #create the model
    m = pretrained_model.Pretrained_Model(shape=(360,640,3), num_outputs=80)
    model = m.build()

    if torch.cuda.is_available(): #send the model to the GPU if available
        model.cuda()

    #configure the training
    criterion = nn.L1Loss()
    optimizer = optim.AdamW(model.parameters(), lr=0.005)
    exp_lr_scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, verbose=True)

    #train the model
    #model = train_model(model, criterion, optimizer, exp_lr_scheduler, num_epochs=480)
    model = train_model(model, criterion, optimizer, exp_lr_scheduler, num_epochs=203)


import datetime
import redis

from django.db import models
from django.db.models import Q
from django.utils import timezone
from django.utils.translation import to_locale, get_language
from django.contrib.auth.models import User
from django.conf import settings
from django.contrib.gis.db.models import PointField
from django_extensions.db.models import (
    TitleSlugDescriptionModel, TimeStampedModel)
from django.utils.translation import gettext_lazy as _

def user_directory_path(instance, filename):
    # file will be uploaded to MEDIA_ROOT / user_<id>/<filename>
    return f'media/{instance.__class__.__name__}/{instance.owner.id}_{filename}'

class AdvertsManager(models.Manager):
    def search(self, query=None):
        qs = self.get_queryset()
        if query is not None:
            or_lookup = (Q(title__icontains=query) |
                         Q(description__icontains=query))
            qs = qs.filter(or_lookup).distinct()  # distinct() is often necessary with Q lookups
        return qs

class Advert(TitleSlugDescriptionModel, TimeStampedModel):
    owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, verbose_name=_('owner'))
    expires = models.DateTimeField(blank=True, null=True, help_text=_('Format mm.dd.yyyy'),
                                   verbose_name=_('expires'))

    LOCALES = (('en', 'en_US'),
               ('uk', 'uk_UA'),
               ('ru', 'ru_RU'),
               ('pl', 'pl_PL'),
               )

    local = models.CharField(
        max_length=2,
        choices=LOCALES,
        default='en'
    )

    objects = AdvertsManager()

    class Meta:
        ordering = ['modified']
        abstract = True

    def was_published_recently(self):
        return self.created >= timezone.now() - datetime.timedelta(days=1)

    def save(self, *args, **kwargs):
        if not self.local:
            lang = get_language()
            if lang:
                self.local = lang[:2]
        super(Advert, self).save(*args, **kwargs)

        r = redis.StrictRedis(host=settings.REDIS_HOST,
                              port=settings.REDIS_PORT,
                              db=settings.REDIS_DB)
        if (self.modified - self.created).seconds == 0:
            r.incr(f'Total:saved')
            r.incr(f'{self.__class__.__name__}:{self.id}:saved')
            data = self.created.timestamp()
            score = f'{self.__class__.__name__}:{self.created.timestamp()}'
            r.zadd("adverts", {score: data})

    def delete(self, using=None, keep_parents=False):
        r = redis.StrictRedis(host=settings.REDIS_HOST,
                              port=settings.REDIS_PORT,
                              db=settings.REDIS_DB)
        super(Advert, self).delete()
        r.decr(f'Total:saved')
        r.decr(f'{self.__class__.__name__}:{self.id}:saved')
        data = self.created.timestamp()
        score = f'{self.__class__.__name__}:{self.created.timestamp()}'
        r.zadd("adverts", {score: data})

    def __str__(self):
        return self.title

class Location(models.Model):
    city = models.CharField(max_length=50, default='Chicago', verbose_name=_('city'))
    address = models.CharField(max_length=100, blank=True, verbose_name=_('address'))
    point = PointField(blank=True, verbose_name=_('map'), null=True)

    # zipcode =

    @property
    def lat_lng(self):
        return list(getattr(self.point, 'coords', [])[::-1])

    class Meta:
        ordering = ['city']
        abstract = True

    def __unicode__(self):
        return self.city

    def __str__(self):
        return self.city


# -*- coding: utf-8 -*-
from distutils.core import setup
from setuptools import find_packages
import re
from os import path

v_file = open(path.join(path.dirname(__file__),
                        'ceda_markup', '__init__.py'))
VERSION = re.compile(r".*__version__ = '(.*?)'",
                     re.S).match(v_file.read()).group(1)

setup(
    name='ceda-markup',
    version=VERSION,
    author=u'Maurizio Nagni',
    author_email='maurizio.nagni',
    packages=find_packages(),
    url='http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/CedaMarkup',
    license='BSD licence, see LICENCE',
    description='Collection of markup classes as geosrss, gml, atom, rss...' + \
                ' Contains an OpenSearch server (just the core not the server)',
    long_description=open('README').read(),
    zip_safe=False,
)

'''
zip_safe=False option. It prevents the package manager to install a
      python egg, instead you'll get a real directory with files in it.
'''


import wx
import time

def LongSimulation(self,input_):
    # Simulate the behavior of the simulation code that I have
    # Actually it returns more than just the progress
    # Occasionally it also returns some partial summary result for diagnostic
    for i in range(0, 10):
        self.log.write('Progress: %3.0f\r' % float(i))
        time.sleep(1)
        wx.GetApp().Yield()
    return 'answer'

class MyForm(wx.Frame):

    def __init__(self):
        wx.Frame.__init__(self, None)

        # Add a panel so it looks the correct on all platforms
        panel = wx.Panel(self, wx.ID_ANY)
        style = wx.TE_MULTILINE|wx.TE_READONLY|wx.HSCROLL

        sizer = wx.BoxSizer(wx.VERTICAL)

        self.log = wx.TextCtrl(panel, wx.ID_ANY, size=(300,100), style=style)
        sizer.Add(self.log, 1, wx.ALL|wx.EXPAND, 5)

        btn = wx.Button(panel, wx.ID_ANY, 'Start')
        self.Bind(wx.EVT_BUTTON, self.onButton, btn)

        sizer.Add(btn, 0, wx.ALL|wx.CENTER, 5)
        panel.SetSizer(sizer)

    def onButton(self, event):
        input_ = 0
        res = LongSimulation(self,input_)
        self.log.write(res)
        self.log.write('\n2222')

# Run the program
if __name__ == "__main__":
    app = wx.App(False)
    frame = MyForm()
    frame.Show()
    app.MainLoop()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import math
import torch.utils.model_zoo as model_zoo

from .blocks import *

BN_MOMENTUM = 0.1

# __all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',
#         'resnet152']

model_urls = {
    'resnet18': 'https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth',
    'resnet34': 'https://s3.amazonaws.com/pytorch/models/resnet34-333f7ec4.pth',
    'resnet50': 'https://s3.amazonaws.com/pytorch/models/resnet50-19c8e357.pth',
    'resnet101': 'https://s3.amazonaws.com/pytorch/models/resnet101-5d3b4d8f.pth',
    'resnet152': 'https://s3.amazonaws.com/pytorch/models/resnet152-b121ed2d.pth',
}

class ResNet(nn.Module):
    def __init__(self, block, layers, num_classes=1000):
        self.inplanes = 64
        super(ResNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
                    bias=False)
        self.bn1 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def _make_layer(self, block, planes, num_blocks, stride=1):
        layers = []
        layers.append(block(self.inplanes, planes, stride))
        self.inplanes = planes * block.expansion
        for i in range(1, num_blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

def resnet18(pretrained=False):
    """Constructs a ResNet-18 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(BasicBlock, [2, 2, 2, 2])
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet18']))
    return model

def resnet34(pretrained=False):
    """Constructs a ResNet-34 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(BasicBlock, [3, 4, 6, 3])
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet34']))
    return model

def resnet50(pretrained=False):
    """Constructs a ResNet-50 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(Bottleneck, [3, 4, 6, 3])
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet50']))
    return model

def resnet101(pretrained=False):
    """Constructs a ResNet-101 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(Bottleneck, [3, 4, 23, 3])
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet101']))
    return model

def resnet152(pretrained=False):
    """Constructs a ResNet-152 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(Bottleneck, [3, 8, 36, 3])
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet152']))
    return model

class deconv_resnet(nn.Module):
    def __init__(self, num_layers=50, num_classes=17, pretrained=False):
        super(deconv_resnet, self).__init__()
        num_feats = 256
        bias = False

        if num_layers == 50:
            self.resnet = resnet50()
            # self.resnet = resnet50(pretrained)
            model_path = 'data/pretrained/resnet50.pth'
        elif num_layers == 101:
            self.resnet = resnet101()
            # self.resnet = resnet101(pretrained)
            model_path = 'data/pretrained/resnet101.pth'
        elif num_layers == 152:
            self.resnet = resnet152()
            # self.resnet = resnet152(pretrained)
            model_path = 'data/pretrained/resnet152.pth'

        if pretrained:
            print("Loading pretrained weights from %s" %(model_path))
            state_dict = torch.load(model_path)
            self.resnet.load_state_dict({k:v for k,v in state_dict.items() if k in self.resnet.state_dict()})

        self.backbone = nn.Sequential(self.resnet.conv1, self.resnet.bn1, self.resnet.relu, self.resnet.maxpool,
                                      self.resnet.layer1, self.resnet.layer2, self.resnet.layer3, self.resnet.layer4)

        self.deconv = nn.Sequential(
                    nn.ConvTranspose2d(2048, num_feats, kernel_size=4, stride=2, padding=1, bias = bias),
                    nn.BatchNorm2d(num_feats, momentum=BN_MOMENTUM),
                    nn.ReLU(inplace=True),
                    nn.ConvTranspose2d(num_feats, num_feats, kernel_size=4, stride=2, padding=1, bias=bias),
                    nn.BatchNorm2d(num_feats, momentum=BN_MOMENTUM),
                    nn.ReLU(inplace=True),
                    nn.ConvTranspose2d(num_feats, num_feats, kernel_size=4, stride=2, padding=1, bias=bias),
                    nn.BatchNorm2d(num_feats, momentum=BN_MOMENTUM),
                    nn.ReLU(inplace=True) )

        self.heatmap = nn.Conv2d(num_feats, num_classes, kernel_size=1)

        for m in self.deconv.modules():
            if isinstance(m, nn.ConvTranspose2d):
                m.weight.data.normal_(0, 0.001)
                # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                # m.weight.data.normal_(0, math.sqrt(2. / n))
                # bound = math.sqrt(6. / n)
                # m.weight.data.uniform_(-bound, bound)
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

        for m in self.heatmap.modules():
            if isinstance(m, nn.Conv2d):
                # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                # m.weight.data.normal_(0, math.sqrt(2. / n))
                m.weight.data.normal_(0, 0.001)
                m.bias.data.zero_()

        def set_bn_fix(m):
            classname = m.__class__.__name__
            if classname.find('BatchNorm') != -1:
                for p in m.parameters(): p.requires_grad=False

        # self.resnet.apply(set_bn_fix)

    def forward(self, x):
        x = self.backbone(x)
        x = self.deconv(x)
        out = self.heatmap(x)

        return out

    # def train(self, mode=True):
    #     # Override train so that the training mode is set as we want
    #     nn.Module.train(self, mode)
    #     if mode:
    #         # Set fixed blocks to be in eval mode (not really doing anything)
    #         self.resnet.eval()
    #         if self.FIXED_BLOCKS <= 3:
    #             self.resnet.layer4.train()
    #         if self.FIXED_BLOCKS <= 2:
    #             self.resnet.layer3.train()
    #         if self.FIXED_BLOCKS <= 1:
    #             self.resnet.layer2.train()
    #         if self.FIXED_BLOCKS == 0:
    #             self.resnet.layer1.train()

    #     # Set batchnorm always in eval mode during training
    #     def set_bn_eval(m):
    #         classname = m.__class__.__name__
    #         if classname.find('BatchNorm') != -1:
    #         m.eval()

    #     self.resnet.apply(set_bn_eval)

    def load_pretrained_resnet(self, state_dict):
        self.resnet.load_state_dict({k: state_dict[k] for k in list(self.resnet.state_dict())})


from __future__ import unicode_literals
import sys
import os
import json
import collections

from pip import cmdoptions

from pip.basecommand import Command
try:
    from pip.log import logger
except ImportError:
    from pip import logger  # 6.0
from pip.index import PackageFinder
from pip.req import RequirementSet, InstallRequirement, parse_requirements
from pip.locations import build_prefix, src_prefix

from .dependency import trace_dependencies
from .version import __version__

def pretty_project_name(req):
    """Get project name in a pretty form:

    name-version

    """
    try:
        print(req.name, req.installed_version)
    except Exception:
        print("dep %s has a problem sire." % req.name)
        return req.name
    return '%s-%s' % (req.name, req.installed_version)

class DependencyChecker(Command):
    bundle = False
    name = 'dependency'

    def __init__(self, *args, **kw):
        super(DependencyChecker, self).__init__(*args, **kw)

        # fix prog name to be gluttony instead of pip dependancy
        self.parser.prog = 'gluttony'

        self.cmd_opts.add_option(cmdoptions.requirements.make())
        self.cmd_opts.add_option(cmdoptions.build_dir.make())
        self.cmd_opts.add_option(cmdoptions.download_cache.make())

        # cmdoptions.editable exist in pip's git
        self.parser.add_option(
            '-e', '--editable',
            dest='editables',
            action='append',
            default=[],
            metavar='VCS+REPOS_URL[@REV]#egg=PACKAGE',
            help='Install a package directly from a checkout. Source will be checked '
            'out into src/PACKAGE (lower-case) and installed in-place (using '
            'setup.py develop). You can run this on an existing directory/checkout (like '
            'pip install -e src/mycheckout). This option may be provided multiple times. '
            'Possible values for VCS are: svn, git, hg and bzr.')

        self.parser.add_option(
            '-d', '--download', '--download-dir', '--download-directory',
            dest='download_dir',
            metavar='DIR',
            default=None,
            help='Download packages into DIR instead of installing them')
        self.parser.add_option(
            '--src', '--source', '--source-dir', '--source-directory',
            dest='src_dir',
            metavar='DIR',
            default=None,
            help='Check out --editable packages into DIR (default %s)' % src_prefix)
        self.parser.add_option(
            '-U', '--upgrade',
            dest='upgrade',
            action='store_true',
            help='Upgrade all packages to the newest available version')
        self.parser.add_option(
            '-I', '--ignore-installed',
            dest='ignore_installed',
            action='store_true',
            help='Ignore the installed packages (reinstalling instead)')

        # options for output
        self.parser.add_option(
            '--dump',
            dest='dump',
            metavar='FILE',
            help='dump dependancy by level')
        self.parser.add_option(
            '-j', '--json',
            dest='json_file',
            metavar='FILE',
            help='JSON filename for result output')
        self.parser.add_option(
            '--pydot',
            dest='py_dot',
            metavar='FILE',
            help='Output dot file with pydot')
        self.parser.add_option(
            '--pygraphviz',
            dest='py_graphviz',
            metavar='FILE',
            help='Output dot file with PyGraphviz')
        self.parser.add_option(
            '--display', '--display-graph',
            dest='display_graph',
            action='store_true',
            help='Display graph with Networkx and matplotlib')
        self.parser.add_option(
            '-R', '--reverse',
            dest='reverse',
            action='store_true',
            help='Reverse the direction of edge')

        index_opts = cmdoptions.make_option_group(
            cmdoptions.index_group,
            self.parser,
        )

        self.parser.insert_option_group(0, index_opts)

    def _build_package_finder(self, options, index_urls, session):
        """
        Create a package finder appropriate to this install command.
        This method is meant to be overridden by subclasses, not
        called directly.
        """
        return PackageFinder(
            use_wheel=False,
            find_links=options.find_links,
            index_urls=index_urls,
            allow_external=options.allow_external,
            allow_unverified=options.allow_unverified,
            allow_all_external=options.allow_all_external,
            session=session,
        )

    def run(self, options, args):
        if not options.build_dir:
            options.build_dir = build_prefix
        if not options.src_dir:
            options.src_dir = src_prefix
        if options.download_dir:
            options.no_install = True
            options.ignore_installed = True
        else:
            options.build_dir = os.path.abspath(options.build_dir)
            options.src_dir = os.path.abspath(options.src_dir)
        session = self._build_session(options)
        index_urls = [options.index_url] + options.extra_index_urls
        if options.no_index:
            logger.notify('Ignoring indexes: %s' % ','.join(index_urls))
            index_urls = []
        finder = self._build_package_finder(options, index_urls, session)
        requirement_set = RequirementSet(
            build_dir=options.build_dir,
            src_dir=options.src_dir,
            download_dir=options.download_dir,
            download_cache=options.download_cache,
            upgrade=options.upgrade,
            ignore_installed=options.ignore_installed,
            ignore_dependencies=False,
            session=session,
        )

        for name in args:
            requirement_set.add_requirement(
                InstallRequirement.from_line(name, None))
        for name in options.editables:
            requirement_set.add_requirement(
                InstallRequirement.from_editable(name, default_vcs=options.default_vcs))
        for filename in options.requirements:
            for req in parse_requirements(filename, finder=finder, options=options):
                requirement_set.add_requirement(req)

        requirement_set.prepare_files(
            finder,
            force_root_egg_info=self.bundle,
            bundle=self.bundle,
        )

        return requirement_set

    def _output_json(self, json_file, dependencies):
        packages = set()
        json_deps = []
        for src, dest in dependencies:
            packages.add(src)
            packages.add(dest)
            json_deps.append([
                pretty_project_name(src),
                pretty_project_name(dest),
            ])

        json_packages = []
        for package in packages:
            json_packages.append(dict(
                name=package.name,
                installed_version=package.installed_version,
            ))

        with open(json_file, 'wt') as jfile:
            json.dump(dict(
                packages=json_packages,
                dependencies=json_deps,
            ), jfile, sort_keys=True, indent=4, separators=(',', ': '))

    def check_conflicts(self, dependencies):
        dependancies_flattened = collections.defaultdict(set)
        for dep1, dep2 in dependencies:
            try:
                if dep1.installed_version is not None:
                    dependancies_flattened[dep1.name].add(dep1.installed_version)
            except Exception:
                print("%s has an unknown version" % dep1.name)

            try:
                if dep2.installed_version is not None:
                    dependancies_flattened[dep2.name].add(dep2.installed_version)
            except Exception:
                print("%s has an unknown version" % dep2.name)

        for dependency_name, dependency_versions in dependancies_flattened.items():
            if dependency_versions and len(dependency_versions) > 1:
                print("Warning: This project requires %s in multiple versions:" % dependency_name, ",".join(dependency_versions))

    def output(self, options, args, dependencies):
        """Output result

        """

        if options.reverse:
            dependencies = map(lambda x: x[::-1], dependencies)

        if options.json_file:
            self._output_json(options.json_file, dependencies)
            logger.notify("Dependencies relationships result is in %s now",
                          options.json_file)

        self.check_conflicts(dependencies)

        if options.display_graph or options.py_dot or options.py_graphviz or options.dump:
            import networkx as nx

            # extract name and version
            def convert(pair):
                return (
                    pretty_project_name(pair[0]),
                    pretty_project_name(pair[1]),
                )
            plain_dependencies = map(convert, dependencies)
            dg = nx.DiGraph()
            dg.add_edges_from(plain_dependencies)

            if options.dump:
                dependancies_ordered = []
                for n, nbrs in dg.adjacency_iter():
                    for nbr, eattr in nbrs.items():
                        dependancies_ordered.append(nbr)

                dependancies_ordered = set(dependancies_ordered)
                with open(options.dump, mode='wt') as myfile:
                    myfile.write('\n'.join(dependancies_ordered))

            if options.py_dot:
                logger.notify("Writing dot to %s with Pydot ...",
                              options.py_dot)
                from networkx.drawing.nx_pydot import write_dot
                write_dot(dg, options.py_dot)
            if options.py_graphviz:
                logger.notify("Writing dot to %s with PyGraphviz ...",
                              options.py_graphviz)
                from networkx.drawing.nx_agraph import write_dot
                write_dot(dg, options.py_graphviz)
            if options.display_graph:
                import matplotlib.pyplot as plt
                logger.notify("Drawing graph ...")

                if not plain_dependencies:
                    logger.notify("There is no dependency to draw.")
                else:
                    pydot_graph = nx.drawing.to_pydot(dg)
                    pydot_graph.write_png("dependency.png")

    def main(self, args):
        options, args = self.parser.parse_args(args)
        if not args:
            self.parser.print_help()
            return

        level = 1  # Notify
        logger.level_for_integer(level)
        logger.consumers.extend([(level, sys.stdout)])
        # get all files
        requirement_set = self.run(options, args)
        # trace dependencies
        logger.notify("Tracing dependencies ...")
        dependencies = []
        values = None
        if hasattr(requirement_set.requirements, 'itervalues'):
            values = list(requirement_set.requirements.itervalues())
        elif hasattr(requirement_set.requirements, 'values'):
            values = list(requirement_set.requirements.values())
        for req in values:
            trace_dependencies(req, requirement_set, dependencies)
        # output the result
        logger.notify("Output result ...")
        self.output(options, args, dependencies)
        requirement_set.cleanup_files()

def main():
    command = DependencyChecker()
    command.main(sys.argv[1:])

if __name__ == '__main__':
    main()

{'nx': 'networkx', 'write_dot': 'networkx.drawing.nx_agraph.write_dot', 'plt': 'matplotlib.pyplot'}


# coding: utf-8

# ### Notebook to configure model

# In[1]:

import time
import math
import copy

import numpy as np
import pandas as pd

import matplotlib
import matplotlib.pyplot as plt

import seaborn as sns
sns.set_context(context="talk")

import torch
import torch.nn as nn
import tensorflow as tf
import torch.nn.functional as F
from torchvision import datasets
import torchvision.transforms as transforms
from torch.autograd import Variable
from torch.utils import data

# In[2]:

isRetrain = False

# Hyperparameter

QUERY_SIZE = 10
EMBEDDING_SIZE = 50

# HIDDEN_SIZE = 512
# ATTENTION_SIZE = 64

parameter_name = 'lr5em2_lm1em4_batch100_q1_e50'

LEARNING_RATE = 5*1e-2
BATCH_SIZE = 100
LAMBDA = 1e-4

num_epochs = 100

VOCAB_SIZE = 5
NUM_CLASSES = 2

# Data-specific

READ_LENGTH = 100

# # KRAS
# GENOME_START = 25204789
# GENOME_END = 25250936

# # NOTCH1
# GENOME_START = 136494433
# GENOME_END = 136545786

# NRAS
file_name = 'NRAS-germline'
GENOME_START = 114704464
GENOME_END = 114716894

GENOME_LENGTH = GENOME_END - GENOME_START + 1
CONTEXT_SIZE = GENOME_LENGTH

ref_names = ["class", "ref", "sequence"]
VOCAB = ['N','A','T','C','G']

# In[3]:

def load_ref_data(file_name, sample_ratio= 1, n_class=2, names=ref_names):

    csv_file = pd.read_csv(file_name, names=ref_names)
    shuffle_csv = csv_file.sample(frac=sample_ratio).reset_index()
#     x = pd.Series(shuffle_csv["sequence"])
    x = list(shuffle_csv["sequence"])
#     ref = pd.Series(shuffle_csv["ref"])
    ref = list(shuffle_csv["ref"])
    y = pd.Series(shuffle_csv["class"])
    y = to_one_hot(y, n_class)
    print(y.shape)
#     print(type(x))
#     print(type(y))
#     print(type(ref))

    return x, ref, y

# In[4]:

def create_synthetic_data(file_name, cancer_genes, benign_genes, num_patients=10, num_reads_per_patients=3, read_length=READ_LENGTH, genome_length=GENOME_LENGTH, vocab=VOCAB, isSomatic=True, print_seq=False):

    seq_list = np.random.choice(vocab, [num_patients, genome_length], replace=True)
    backup_seq_list = seq_list

    for loc, mutation in cancer_genes.items():
        seq_list[np.random.choice(num_patients, int(num_patients*mutation[1]), replace=False), loc] = mutation[0]

    genomes = []
    for r in range(seq_list.shape[0]):
        seq = ''.join(seq_list[r,:])
        if print_seq:
            print(seq)
        genomes.append(seq)

    locs = np.random.choice(genome_length-read_length, num_patients*num_reads_per_patients)

    file = open('./tumor-genome-'+file_name+'.txt','w')
    count = 0
    reads = []
    for genome in genomes:
        for t in range(num_reads_per_patients):
            index = count*num_reads_per_patients+t
            reads.append(genome[locs[index]:locs[index]+read_length])
        file.write("%s\n" % genome)
        count = count + 1
    file.close()

    file = open('./tumor-syn-'+file_name+'.csv','w')
    for r in range(num_patients*num_reads_per_patients):
        file.write("1, %d, %s\n" % (locs[r], reads[r]))
    file.close()

    tumor_locs = locs
    tumor_reads = reads

    if isSomatic:
        seq_list = backup_seq_list
    else:
        seq_list = np.random.choice(vocab, [num_patients, genome_length], replace=True)

    for loc, mutation in benign_genes.items():
        seq_list[np.random.choice(num_patients, int(num_patients*mutation[1]), replace=False), loc] = mutation[0]

    genomes = []
    for r in range(seq_list.shape[0]):
        seq = ''.join(seq_list[r,:])
        if print_seq:
            print(seq)
        genomes.append(seq)

    locs = np.random.choice(genome_length-read_length, num_patients*num_reads_per_patients)

    file = open('./normal-genome-'+file_name+'.txt','w')
    count = 0
    reads = []
    for genome in genomes:
        for t in range(num_reads_per_patients):
            index = count*num_reads_per_patients+t
            reads.append(genome[locs[index]:locs[index]+read_length])
        file.write("%s\n" % genome)
        count = count + 1
    file.close()

    file = open('./normal-syn-'+file_name+'.csv','w')
    for r in range(num_patients*num_reads_per_patients):
        file.write("0, %d, %s\n" % (locs[r], reads[r]))
    file.close()

    normal_locs = locs
    normal_reads = reads

    file = open('./syn-'+file_name+'.csv','w')
    for r in range(num_patients*num_reads_per_patients):
        file.write("1,%d,%s\n" % (tumor_locs[r], tumor_reads[r]))
        file.write("0,%d,%s\n" % (normal_locs[r], normal_reads[r]))
    file.close()

    return './syn-'+file_name+'.csv'

# In[5]:

def to_one_hot(y, n_class):

    return np.eye(n_class)[y.astype(int)]

# In[6]:

def split_ref_dataset(x_test, y_test, ref_test, dev_ratio):

    test_size = len(x_test)
    print(test_size)
    dev_size = (int)(test_size * dev_ratio)
    print(dev_size)

    x_dev = x_test[:dev_size]
    x_test = x_test[dev_size:]
    y_dev = y_test[:dev_size]
    y_test = y_test[dev_size:]
    ref_dev = ref_test[:dev_size]
    ref_test = ref_test[dev_size:]

    return x_test, x_dev, y_test, y_dev, ref_test, ref_dev, dev_size, test_size - dev_size

# In[7]:

class TensorizedReadDataset(torch.utils.data.DataLoader):
    'Characterizes a Tensorized dataset for genome reads in PyTorch'

    def __init__(self, reads, ref_locs, labels, read_length=100, genome_start=0, genome_end=0):
#         super(TensorizedReadDataset, self).__init__()

        # self.read_length = read_length
        self.labels = labels
        self.reads = reads
        self.ref_locs = ref_locs
        self.genome_start = genome_start
        self.genome_end = genome_end

    def __len__(self):
        return len(self.reads)

    def __getitem__(self, index):

        vals = list(self.reads[index])
        read_length = len(vals)
        locs = list(np.arange(self.ref_locs[index]-self.genome_start,self.ref_locs[index]+read_length-self.genome_start))

#         print(len(vals))
#         print(len(locs))

        vals2idx = {'N': 0, 'A': 1, 'C': 2, 'T': 3, 'G': 4}
#         read = torch.LongTensor(np.array([vals2idx[val]+loc*len(vals2idx) for val, loc in zip(vals, locs)], dtype=int), requires_grad=False)

        read = torch.autograd.Variable(torch.LongTensor(np.array([vals2idx[val]+loc*len(vals2idx) for val, loc in zip(vals, locs)], dtype=int)), requires_grad=False)

        X = read
        Y = self.labels[index,:]

#         torch.LongTensor(self.labels[index,:])

        return X, Y

# In[8]:

class SequenceAttentionClassifier(nn.Module):

    def __init__(self, genome_length, vocab_size=5, query_size=10, embedding_size=128, num_classes=2):

        super(SequenceAttentionClassifier, self).__init__()

        self.genome_length = genome_length
        self.vocab_size = vocab_size
        self.query_size = query_size
        self.embedding_size = embedding_size
        self.num_classes = num_classes
        self.K = nn.Embedding(vocab_size*genome_length, embedding_size)
        self.V = nn.Embedding(vocab_size*genome_length, query_size)
        self.W = nn.Linear(query_size, num_classes)
        self.Q = nn.Linear(embedding_size, query_size)

    def forward(self, read):

        # 'read' here should be mapped to a flattened form where X_ij = 1 maps to i*vocab_size + j
        K_lookup = self.K(read) # Get the relevant keys
        V_lookup = self.V(read) # Get the relevant values

        # Get the attention weights
        logits = self.Q(K_lookup) / math.sqrt(self.embedding_size)
        probs = F.softmax(logits, dim = 0)

        # Calculate the covariates for the logistic regression
#         X = torch.matmul(probs.transpose(1,2), V_lookup)
#         X = probs * V_lookup
        X = (probs * V_lookup).sum(dim=1)

        # Right now we can just ignore the fact that we're doing a linear-transform.
        # In the future we'll add nonlinearities

        # Return the logits for the classifier
        return self.W(X), K_lookup, V_lookup

# In[9]:

def GetTensorBatch(reads, ref_locs, labels, batch_size=100, genome_start=0, genome_end=0):

    batches = {}
    set_size = len(ref_locs)
    for batch in range(set_size // batch_size):

        x_batch = []
        y_batch = []

        for index in range(batch_size):

            vals = list(reads[index])
            read_length = len(vals)
            locs = list(np.arange(ref_locs[index]-genome_start,ref_locs[index]+read_length-genome_start))

            vals2idx = {'N': 0, 'A': 1, 'C': 2, 'T': 3, 'G': 4}
            read = torch.autograd.Variable(torch.LongTensor(np.array([vals2idx[val]+loc*len(vals2idx) for val, loc in zip(vals, locs)], dtype=int)), requires_grad=False)

            X = read
            Y = labels[index,:]
            x_batch.append(X)
            y_batch.append(Y)

        batches[batch] = [x_batch, y_batch]

    return batches

# In[14]:

# load data

x_train, refs_train, y_train = load_ref_data("../data/ref-germline-NRAS-train39000.csv", sample_ratio=1)
x_hardtest, refs_hardtest, y_hardtest = load_ref_data("../data/ref-germline-NRAS-test1000.csv", sample_ratio=1)

# split dataset to train and validation
x_train, x_test, y_train, y_test, refs_train, refs_test, test_size, train_size = split_ref_dataset(x_train, y_train, refs_train, 0.2)

# split dataset to test and dev
x_softtest, x_val, y_softtest, y_val, refs_softtest, refs_val, val_size, softtest_size = split_ref_dataset(x_test, y_test, refs_test, 0.5)

print("Training size: ", train_size)
print("Soft Test size: ", softtest_size)
print("Hard Test size: ", len(y_hardtest))
print("Validation size: ", val_size)

# In[15]:

# Generators
# train_dataset = TensorizedReadDataset(reads=x_train,
#                                       ref_locs=refs_train,
#                                       labels=y_train,
#                                       read_length=READ_LENGTH,
#                                       genome_start=GENOME_START,
#                                       genome_end=GENOME_END)

# hardval_dataset = TensorizedReadDataset(reads=x_test,
#                                         ref_locs=refs_test,
#                                         labels=y_test,
#                                         read_length=READ_LENGTH,
#                                         genome_start=GENOME_START,
#                                         genome_end=GENOME_END)

# softval_dataset = TensorizedReadDataset(reads=x_softval,
#                                         ref_locs=refs_softval,
#                                         labels=y_softval,
#                                         read_length=READ_LENGTH,
#                                         genome_start=GENOME_START,
#                                         genome_end=GENOME_END)

# # Input pipeline
# train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
#                                            batch_size=BATCH_SIZE,
#                                            shuffle=True)

# hardtest_loader = torch.utils.data.DataLoader(dataset=hardval_dataset,
#                                              batch_size=BATCH_SIZE,
#                                              shuffle=True)

# softval_loader = torch.utils.data.DataLoader(dataset=softval_dataset,
#                                              batch_size=BATCH_SIZE,
#                                              shuffle=True)

train_loader = GetTensorBatch(reads=x_train,
                                      ref_locs=refs_train,
                                      labels=y_train,
                                      batch_size=BATCH_SIZE,
                                      genome_start=GENOME_START,
                                      genome_end=GENOME_END)

val_loader = GetTensorBatch(reads=x_val,
                                      ref_locs=refs_val,
                                      labels=y_val,
                                      batch_size=BATCH_SIZE,
                                      genome_start=GENOME_START,
                                      genome_end=GENOME_END)

hardtest_loader = GetTensorBatch(reads=x_hardtest,
                                        ref_locs=refs_hardtest,
                                        labels=y_hardtest,
                                        batch_size=BATCH_SIZE,
                                        genome_start=GENOME_START,
                                        genome_end=GENOME_END)

softtest_loader = GetTensorBatch(reads=x_softtest,
                                        ref_locs=refs_softtest,
                                        labels=y_softtest,
                                        batch_size=BATCH_SIZE,
                                        genome_start=GENOME_START,
                                        genome_end=GENOME_END)

# In[16]:

# isRetrain = True

# In[17]:

model = SequenceAttentionClassifier(genome_length=GENOME_LENGTH,

                                    vocab_size=VOCAB_SIZE,
                                    query_size=QUERY_SIZE,
                                    embedding_size=EMBEDDING_SIZE,
                                    num_classes=NUM_CLASSES)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE)

if isRetrain:
    model.load_state_dict(torch.load('./'+file_name+'_model.pth'))

# In[18]:

f1 = open('cv_acc_each_epoch_'+parameter_name+'.txt', 'a+')
f2 = open('cv_loss_each_batch_'+parameter_name+'.txt', 'a+')
f3 = open('cv_loss_val_each_batch_'+parameter_name+'.txt', 'a+')
f4 = open('cv_acc_each_batch_'+parameter_name+'.txt', 'a+')

lam = LAMBDA

# Training process
for epoch in range(num_epochs):
#     b = 0 # count batch
    for b, [x_batch, y_batch] in train_loader.items():

        full_loss = 0
        full_data_loss = 0
        optimizer.zero_grad()
        for c in range(BATCH_SIZE):

            x_input = x_batch[c].view(1, x_batch[c].shape[0])
            y_input = torch.Tensor(y_batch[c]).type(torch.float64).view(1, y_batch[c].shape[0])

            outputs, K, V = model(x_input)
            data_loss = criterion(outputs, torch.max(y_input, 1)[1])

            reg_loss = (torch.norm(K,2,1).mean() + torch.norm(V,2,1).mean())*lam
            loss = data_loss + reg_loss
            full_loss = full_loss + loss
            full_data_loss = full_data_loss + data_loss

            loss.backward()
            optimizer.step()
            f2.write(repr(data_loss)+"\n")

#         full_loss = full_loss / 2
        # full_loss.backward()
        # optimizer.step()

        # if (b + 1) % 1 == 0:
        print("Epoch {}, Batch {}, loss :{}".format(epoch + 1, b + 1, full_loss))
#         b = b + 1

        # loss_each_batch.append(full_loss)
        # f2.write(repr(full_data_loss)+"\n")

        full_loss = 0
        full_data_loss = 0

        correct = 0
        total = 0
        for b, [x_batch, y_batch] in val_loader.items():
            full_vloss = 0
            for c in range(BATCH_SIZE):

                x_input = x_batch[c].view(1, x_batch[c].shape[0])
                y_input = torch.Tensor(y_batch[c]).type(torch.float64).view(1, y_batch[c].shape[0])

                outputs, K, V = model(x_input)
                _, predicted = torch.max(outputs.data, 1)

                data_vloss = criterion(outputs, torch.max(y_input, 1)[1])
                full_vloss = full_vloss + data_vloss

                total += len(y_input)
                correct += (predicted == torch.max(y_input.type(torch.LongTensor), 1)[1]).sum()

                f3.write(repr(data_vloss)+"\n")
            # f3.write(repr(full_vloss)+"\n")

        acc = int(correct) / total
        print('Validation Accuracy: {}%'.format(100 * acc))
        # acc_each_epoch.append(acc)
        f4.write(repr(acc)+"\n")

    correct = 0
    total = 0
    for b, [x_batch, y_batch] in val_loader.items():
        for c in range(BATCH_SIZE):

            x_input = x_batch[c].view(1, x_batch[c].shape[0])
            y_input = torch.Tensor(y_batch[c]).type(torch.float64).view(1, y_batch[c].shape[0])

            outputs, K, V = model(x_input)
            _, predicted = torch.max(outputs.data, 1)
            total += len(y_input)
            correct += (predicted == torch.max(y_input.type(torch.LongTensor), 1)[1]).sum()
    acc = int(correct) / total
    print('Validation Accuracy: {}%'.format(100 * acc))
    # acc_each_epoch.append(acc)
    f1.write(repr(acc)+"\n")

f1.close()
f2.close()
f3.close()
f4.close()

# In[202]:

torch.save(model, './full_' + file_name + '_' + parameter_name + '_model.pt')
torch.save(model.state_dict(), './'+file_name + '_' + parameter_name + '_model.pth')

# In[ ]:

# plt.rcParams['figure.figsize'] = [30, 5]

# plt.plot(np.arange(len(loss_each_batch))+1, loss_each_batch)
# plt.xlim(1, len(loss_each_batch))
# plt.title('loss vs. batchs')
# plt.show()

# plt.plot(np.arange(len(acc_each_batch))+1, acc_each_batch)
# plt.xlim(1, len(acc_each_batch))
# plt.title('accuracy vs. batchs')
# plt.show()

# plt.plot(np.arange(len(acc_each_epoch))+1, acc_each_epoch)
# plt.xlim(1, len(acc_each_epoch))
# plt.title('accuracy vs. epochs')
# plt.show()

# In[10]:

# model = torch.load('./full_'+file_name+'_model.pt')
# model.load_state_dict(torch.load('./'+file_name+'_model.pth'))

# In[16]:

correct = 0
total = 0
for b, [x_batch, y_batch] in softtest_loader.items():
    for c in range(BATCH_SIZE):

        x_input = x_batch[c].view(1, x_batch[c].shape[0])
        y_input = torch.Tensor(y_batch[c]).type(torch.float64).view(1, y_batch[c].shape[0])

        outputs, K, V = model(x_input)
        _, predicted = torch.max(outputs.data, 1)
        total += len(y_input)
        correct += (predicted == torch.max(y_input.type(torch.LongTensor), 1)[1]).sum()
acc = int(correct) / total
print('Soft Test Accuracy: {}%'.format(100 * acc))

correct = 0
total = 0
for b, [x_batch, y_batch] in hardtest_loader.items():
    for c in range(BATCH_SIZE):

        x_input = x_batch[c].view(1, x_batch[c].shape[0])
        y_input = torch.Tensor(y_batch[c]).type(torch.float64).view(1, y_batch[c].shape[0])

        outputs, K, V = model(x_input)
        _, predicted = torch.max(outputs.data, 1)
        total += len(y_input)
        correct += (predicted == torch.max(y_input.type(torch.LongTensor), 1)[1]).sum()
acc = int(correct) / total
print('Hard Test Accuracy: {}%'.format(100 * acc))


import logging, fastaResFun
from Bio import SeqIO
import json, sys

def isoformAln(aln, o):
        """Function to cluster isoforms according to the alignment. Return the
                overall coverage of these isoforms.

        Isoforms are from the same species (recognized through keyword
        xxxXxx at the beginning of their name) and same letters or
        indels at same positions in alignment.

        @param1 aln: Path to alignment
        @param2 o: output directory
        @return outAln: Path to file of resulting alignment

        """

        logger = logging.getLogger("main.alignment")
        print("Clustering isoforms.")

        dRem={} #for remaining sequences
        dId2Seq={} #for remaining sequences
        laln=0 #alignement length
        for fasta in SeqIO.parse(open(aln),'fasta'):
                post=fasta.id.find("_")
                if post!=-1: #regular format
                        sp=fasta.id[:post]
                        tag=fasta.id[post+1:]
                        if not sp in dId2Seq:
                                dId2Seq[sp]={}
                        dId2Seq[sp][tag]=str(fasta.seq)
                        if laln==0:
                                laln=len(fasta.seq)
                else:
                        dRem[fasta.id]=str(fasta.seq)

        outCov = o
        clustok=False #flag to check if a cluster has occured
        for sp,dtagseq in dId2Seq.items():
                lclust=[list(dtagseq)] #list of clusters of tags to be split
                for pos in range(laln):
                        lclust2=[]
                        for clust in lclust:
                                dlet={tag:dtagseq[tag][pos] for tag in clust}
                                llet=set([x for x in dlet.values() if x!="-"])
                                if len(llet)<=1: #one letter at most, keep all
                                        lclust2.append(clust)
                                        continue
                                else:
                                        for x in llet:
                                                lclust2.append([tag for tag in clust if dlet[tag]==x])
                                        lind=[tag for tag in clust if dlet[tag]=="-"] #conservative, do not know wether to merge, may be improved
                                        if len(lind)!=0:
                                                lclust2.append(lind)
                        lclust=lclust2

                #now merge sequences in each cluster
                for clust in lclust:
                        if len(clust)==1:
                                dRem[sp+"_"+clust[0]]=dtagseq[clust[0]]
                        else:
                                clustok=True
                                ntag=clust[-1]+"_clust"
                                print("Clustered sequences " + sp+"_" + (", %s_"%(sp)).join(clust) + " into %s_"%(sp)+ntag)
                                nseq="".join([max([dtagseq[tag][pos] for tag in clust]) for pos in range(laln)])
                                dRem[sp+"_"+ntag]=nseq

        if clustok:
            with open(outCov, "w") as outC:
                outC.write(fastaResFun.dict2fasta(dRem))
                outC.close()

                return(outCov)
        else:
                return(aln)


#!/usr/bin/env python3

import sys
import collections
import itertools
import numpy

import aoc

infilename = 'input.txt'
if len(sys.argv) > 2 and sys.argv[1] == '-i':
    infilename = sys.argv[2]

print('Using input file: %s' % infilename)

f = open(infilename, 'r')
data = f.readlines()
f.close()

data = [line.strip() for line in data]

scanners = {}
for ii in range(len(data)):
    line = data[ii]
    if not line:
        continue

    if line.startswith('--'):
        curr = int(line.split()[2])
        scanners[curr] = []
        continue

    scanners[curr].append(list(map(int, line.split(','))))

# compute manhattan distances from each sensor node to all other nodes, compare to
# each sensor and find sets that have 12 of the same distances which are then the
# shared nodes between two sensors
matches=[]
for mm in range(len(scanners)):
    for nn in range(mm+1, len(scanners)):
        if mm == nn:
            continue
        for xx in range(len(scanners[mm])):
            for yy in range(len(scanners[nn])):
                dists0 = [aoc.mandist(scanners[mm][xx], pos) for pos in scanners[mm]]
                dists1 = [aoc.mandist(scanners[nn][yy], pos) for pos in scanners[nn]]
                s = set(dists0).intersection(set(dists1))
                if len(s)>=12:
                    print(mm, nn,xx,yy, s)
                    matches.append((mm, nn, xx, yy))
                    matches.append((nn, mm, yy, xx))

def translate_all(scanners, matches):
    open_set = list(range(len(scanners)))
    open_set.remove(0)
    closed_set = [0]

    beacons = [tuple(t) for t in scanners[0]]

    match_pairs = list(set([(el[0], el[1]) for el in matches]))

    translated = {}
    offsets = {0:0}

    sensor_pos = [(0,0,0)]

    iters = 0
    while open_set:

        # get the next mapping to do - any pair that matches a closed sensor
        # to an open sensor
        pair = None
        for p in match_pairs:
            if p[0] in closed_set and p[1] in open_set:
                pair = p
                break
        assert(pair)

        left = pair[0]
        right = pair[1]
        curr_batch = [el for el in matches if el[0] == left and el[1] == right]

        done = False
        # figure out which orientation results in all of the shared nodes point to the same
        # sensor pos
        for inds in itertools.permutations(range(3),3):
            for signs in itertools.product([-1,1], repeat=3):
                inds = numpy.array(inds)
                signs = numpy.array(signs)

                # found when all shared nodes rotate and offset to the same position relative to sensor 0
                pos = [aoc.tup_add(beacons[offsets[left]+el[2]], numpy.array(scanners[right][el[3]])[inds] * numpy.array(signs)) for el in curr_batch]
                if len(set(pos))==1:
                    #print(pos)
                    sensor_pos.append(pos[0])
                    done = True
                    break
            if done:
                break

        assert(done)

        # translate all of the beacons to the sensor 0 space (shared plus nonshared) and
        # track the offsets so our original mapping from above still works
        offsets[right] = len(beacons)
        trans = [aoc.tup_sub(pos[0], numpy.array(el)[inds] * numpy.array(signs)) for el in scanners[right]]
        beacons.extend(trans)
        translated[right] = trans

        iters += 1

        open_set.remove(right)
        closed_set.append(right)

    return beacons, sensor_pos

all_beacons, sensors = translate_all(scanners, matches)
print(len(sensors), sensors)
print(len(set(all_beacons)))

maxd = 0
for xx, yy in itertools.combinations(range(len(sensors)),2):
    d = aoc.mandist(sensors[xx], sensors[yy])
    if d > maxd:
        maxd = d
print(maxd)


# -*- coding: utf-8 -*-
"""
Created on Fri Sep 29 11:30:46 2017

@author: Deepak

Part A - Network A
"""

import sys
sys.path.append('../../')
sys.path.append('../')
import tensorflow as tf
from fdl_examples.datatools import input_data
import scipy
#import numpy as np
#import time, shutil, os
#import matplotlib.pyplot as plt
#from scipy import misc

# read in MNIST data --------------------------------------------------
mnist = input_data.read_data_sets("../../data/", one_hot=True)

# run network ----------------------------------------------------------

# Parameters
learning_rate = 0.01
training_epochs = 50 # NOTE: you'll want to eventually change this
batch_size = 100
display_step = 1

def inference(x,W,b):
    output = tf.nn.softmax(tf.matmul(x, W) + b)
    w_hist = tf.summary.histogram("weights", W)
    b_hist = tf.summary.histogram("biases", b)
    y_hist = tf.summary.histogram("output", output)
    return output

def loss(output, y):
    dot_product = y * tf.log(output)
    # Reduction along axis 0 collapses each column into a single
    # value, whereas reduction along axis 1 collapses each row
    # into a single value. In general, reduction along axis i
    # collapses the ith dimension of a tensor to size 1.
    xentropy = -tf.reduce_sum(dot_product, axis=1)
    loss = tf.reduce_mean(xentropy)
    return loss

def training(cost, global_step):
    tf.summary.scalar("cost", cost)
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    train_op = optimizer.minimize(cost, global_step=global_step)
    return train_op

def evaluate(output, y):
    correct_prediction = tf.equal(tf.argmax(output, 1), tf.argmax(y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    tf.summary.scalar("validation error", (1.0 - accuracy))
    return accuracy

if __name__ == '__main__':
#    if os.path.exists("logistic_logs/"):
#        shutil.rmtree("logistic_logs/")

    with tf.Graph().as_default():
        x = tf.placeholder("float", [None, 784]) # mnist data image of shape 28*28=784
        y = tf.placeholder("float", [None, 10]) # 0-9 digits recognition => 10 classes
        init = tf.constant_initializer(value=0)
        W = tf.get_variable("W", [784, 10],
                             initializer=init)
        b = tf.get_variable("b", [10],
                             initializer=init)
        output = inference(x,W,b)
        cost = loss(output, y)
        global_step = tf.Variable(0, name='global_step', trainable=False)
        train_op = training(cost, global_step)
        eval_op = evaluate(output, y)
        summary_op = tf.summary.merge_all()
        saver = tf.train.Saver()
        sess = tf.Session()

 #       summary_writer = tf.summary.FileWriter("logistic_logs/",
 #                                           graph_def=sess.graph_def)

        init_op = tf.global_variables_initializer()
        sess.run(init_op)

        # PLOTTING EACH DIGIT
        #x[7]=0, x[6]=1, x[13]=2, x[1]=3, x[2]=4, x[27]=5, x[26]=6, x[25]=7, x[9]=8, x[8]=9
        mini_x, mini_y = mnist.train.next_batch(30)

        num=mini_x[7]
        num0=num.reshape(28,28)

        num=mini_x[6]
        num1=num.reshape(28,28)

        num=mini_x[13]
        num2=num.reshape(28,28)

        num=mini_x[1]
        num3=num.reshape(28,28)

        num=mini_x[2]
        num4=num.reshape(28,28)

        num=mini_x[27]
        num5=num.reshape(28,28)

        num=mini_x[26]
        num6=num.reshape(28,28)

        num=mini_x[25]
        num7=num.reshape(28,28)

        num=mini_x[9]
        num8=num.reshape(28,28)

        num=mini_x[8]
        num9=num.reshape(28,28)

#       plt.imshow(num1)
#
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/0.jpg', num0)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/1.jpg', num1)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/2.jpg', num2)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/3.jpg', num3)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/4.jpg', num4)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/5.jpg', num5)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/6.jpg', num6)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/7.jpg', num7)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/8.jpg', num8)
        scipy.misc.imsave('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/9.jpg', num9)

#        scipy.misc.imsave('outfile.jpg', image_array)

        # Training cycle
        for epoch in range(training_epochs):
            avg_cost = 0.
            total_batch = int(mnist.train.num_examples/batch_size)
            # Loop over all batches
            for i in range(batch_size):
                minibatch_x, minibatch_y = mnist.train.next_batch(batch_size)
            # Fit training using batch data
            sess.run(train_op, feed_dict={x: minibatch_x, y: minibatch_y})
            # Compute average loss
            avg_cost += sess.run(cost, feed_dict={x: minibatch_x, y: minibatch_y})/total_batch
            # Display logs per epoch step
            if epoch % display_step == 0:
                print("Epoch:", '%04d' % (epoch+1), "cost =", "{:.9f}".format(avg_cost))
                accuracy = sess.run(eval_op, feed_dict={x: mnist.validation.images, y: mnist.validation.labels})
                print("Validation Error:", (1 - accuracy))
                summary_str = sess.run(summary_op, feed_dict={x: minibatch_x, y: minibatch_y})
                #summary_writer.add_summary(summary_str, sess.run(global_step))
                #saver.save(sess, "logistic_logs/model-checkpoint", global_step=global_step)
        print("Optimization Finished!")
        accuracy = sess.run(eval_op, feed_dict={x: mnist.test.images, y: mnist.test.labels})

        #PLOTTING FIRST 10 WEIGHTS
        for wi in range (9):
            weights=sess.run(W)
            im=weights[:,wi]
            wim=im.reshape(28,28)
            filename = "C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/W_%d.jpg"%wi
            scipy.misc.imsave(filename, wim)
        print("Test Accuracy:", accuracy)

        with open('C:/Users/Deepak/Dropbox/Deep Learning/Project 2/Part A1 Output/Accuracy.txt', 'w') as f:
            print('Accuracy = ',accuracy, file=f)
        f.close()


import sys
import os
import time
import tempfile

import arcade
import pyglet

from arcade.color import RED

from ev3dev2simulator.state import WorldState
from ev3dev2simulator.visualisation.Sidebar import Sidebar
from ev3dev2simulator.config.config import get_simulation_settings, debug

def start():
    arcade.run()

def get_screens():
    display = pyglet.canvas.get_display()
    screens = display.get_screens()
    return screens

class Visualiser(arcade.Window):
    """
    Main simulator class.
    This class extends from arcade.Window and manages the updates and rendering of the simulator window.
    """

    def __init__(self, update_world_cb, world_state: WorldState, show_fullscreen: bool,
                 show_maximized: bool, use_second_screen_to_show_simulator: bool):

        self.pid_file = None
        self.pid = None
        self.check_for_unique_instance()
        self.update_callback = update_world_cb

        self.sidebar = None
        self.world_state = world_state

        self.current_screen_index = None
        self.set_screen_to_display_simulator_at_startup(use_second_screen_to_show_simulator)

        sim_settings = get_simulation_settings()

        self.scale = None
        self.screen_width = int(sim_settings['screen_settings']['screen_width'])
        self.screen_height = int(sim_settings['screen_settings']['screen_height'])
        self.side_bar_width = int(sim_settings['screen_settings']['side_bar_width'])

        from ev3dev2.version import __version__ as api_version
        from ev3dev2simulator.version import __version__ as sim_version
        screen_title = sim_settings['screen_settings'][
                           'screen_title'] + f'          version: {sim_version}      ev3dev2 api: {api_version}'

        self.frames_per_second = sim_settings['exec_settings']['frames_per_second']
        self.falling_msg = sim_settings['screen_settings']['falling_message']
        self.restart_msg = sim_settings['screen_settings']['restart_message']

        self.change_scale(self.screen_width, self.screen_height)
        if debug:
            print('starting simulation with scaling', self.scale)
            print('arcade version: ', arcade.version.VERSION)

        super(Visualiser, self).__init__(self.screen_width, self.screen_height, screen_title, update_rate=1 / 30,
                                         resizable=True)

        icon1 = pyglet.image.load(r'assets/images/body.png')
        self.set_icon(icon1)
        arcade.set_background_color(eval(sim_settings['screen_settings']['background_color']))

        self.msg_counter = 0

        self.setup_sidebar()
        self.world_state.setup_pymunk_shapes(self.scale)
        self.world_state.setup_visuals(self.scale)

        if show_fullscreen:
            self.toggleFullScreenOnCurrentScreen()

        if show_maximized:
            self.maximize()

        self.check_for_activation()

    @property
    def msg_x(self):
        return (self.screen_width - self.side_bar_width) / 2

    def change_scale(self, new_screen_width, new_screen_height):
        x_scale = (new_screen_width - self.side_bar_width) / self.world_state.board_width
        y_scale = new_screen_height / self.world_state.board_height
        if x_scale <= y_scale:
            scale = x_scale
        else:
            scale = y_scale
        self.screen_height = int(scale * self.world_state.board_height)
        self.screen_width = self.side_bar_width + int(scale * self.world_state.board_width)
        self.scale = scale

    def setup_sidebar(self):
        self.sidebar = Sidebar(self.screen_width - self.side_bar_width, self.screen_height - 70,
                               self.side_bar_width, self.screen_height)
        for robot in self.world_state.get_robots():
            self.sidebar.init_robot(robot.name, robot.sensors, robot.bricks, robot.side_bar_sprites)

    def set_screen_to_display_simulator_at_startup(self, use_second_screen_to_show_simulator):
        """ Set screen to use to display the simulator at startup. For windows this works only in fullscreen mode.

           By default set current screen to show simulator, but if use_second_screen_to_show_simulator==True
           then change screen to other screen.

           On MacOS this works for both fullscreen and none-fullscreen mode.
           On Windows this only works for fullscreen mode. For none-fullscreen always the first screen is used.
        """

        # get current_screen_index
        current_screen_index = 1 if use_second_screen_to_show_simulator else 0
        screens = get_screens()
        # for screen in screens: print(screen)
        num_screens = len(screens)
        if num_screens == 1:
            current_screen_index = 0
        self.current_screen_index = current_screen_index

        # change screen to show simulator
        # HACK override default screen function to change it.
        # Note: arcade window class doesn't has the screen parameter which pyglet has, so by overriding
        #       the get_default_screen method we can still change the screen parameter.
        def get_default_screen():
            """Get the default screen as specified by the user's operating system preferences."""
            return screens[self.current_screen_index]

        display = pyglet.canvas.get_display()
        display.get_default_screen = get_default_screen

        # note:
        #  for macOS  get_default_screen() is also used to as the screen to draw the window initially
        #  for windows the current screen is used to to draw the window initially,
        #              however the value set by get_default_screen() is used as the screen
        #              where to display the window fullscreen!

        # note:  BUG: dragging window to other screen in macOS messes up view size
        #   for macOS the screen of the mac can have higher pixel ratio (self.get_pixel_ratio())
        #   then the second screen connected. If you drag the window from the mac screen to the
        #   second screen then the windows may be the same size, but the simulator is drawn in only
        #   in the lower left quart of the window.
        #      => we need somehow make drawing of the simulator larger

        # how to view simulator window on second screen when dragging not working?
        #    SOLUTION: just when starting up the simulator set it to open on the second screen,
        #              then it goes well, and you can also open it fullscreen on the second screen
        # see also : https://stackoverflow.com/questions/49302201/highdpi-retina-windows-in-pyglet

    def check_for_unique_instance(self):
        """ Detect whether an other instance is already running. If so then trigger the
            activation for the other instance and terminate this instance.
        """

        tmpdir = tempfile.gettempdir()
        self.pid_file = os.path.join(tmpdir, "ev3dev2simulator.pid")

        self.pid = str(os.getpid())
        f = open(self.pid_file, 'w')
        f.write(self.pid)
        f.flush()
        f.close()

        time.sleep(2)

        file = open(self.pid_file, 'r')
        line = file.readline()
        file.close()
        read_pid = line.rstrip()
        if read_pid != self.pid:
            # other process already running
            sys.exit()

    def check_for_activation(self):
        """ checks each interval whether the simulator windows must be activated (bring to front)

            note: activation can happen when one tries to start another instance of the simulator,
                  and that instance detects an instance is already running. It then triggers the
                  activation for the other instance and terminates itself.
        """
        from pyglet import clock

        def callback(_):
            file = open(self.pid_file, 'r')
            line = file.readline()
            file.close()
            read_pid = line.rstrip()
            if read_pid != self.pid:

                # other simulator tries to start running
                # write pid to pid_file to notify this simulator is already running
                f = open(self.pid_file, 'w')
                f.write(self.pid)
                f.close()

                import platform
                if platform.system().lower().startswith('win'):
                    self.windows_activate()
                else:
                    self.activate()

        clock.schedule_interval(callback, 1)

    def windows_activate(self):
        # noinspection PyProtectedMember
        from pyglet.libs.win32 import _user32
        from pyglet.libs.win32.constants import SW_SHOWMINIMIZED, SW_SHOWNORMAL
        _user32.ShowWindow(self._hwnd, SW_SHOWMINIMIZED)
        _user32.ShowWindow(self._hwnd, SW_SHOWNORMAL)

    def on_close(self):
        sys.exit(0)

    def on_key_press(self, key, modifiers):
        """Called whenever a key is pressed. """

        # Quit the simulator
        if key == arcade.key.Q:
            self.on_close()

        # Toggle fullscreen between screens (only works at fullscreen mode)
        elif key == arcade.key.T:
            # User hits T. When at fullscreen, then switch screen used for fullscreen.
            if len(get_screens()) == 0:
                return
            if self.fullscreen:
                # to switch screen when in fullscreen we first have to back to normal window, and do fullscreen again
                self._set_full_screen(False)
                # Toggle between first and second screen (other screens are ignored)
                self.toggle_screen_used_for_fullscreen()
                self._set_full_screen(True)

        # Maximize window
        # note: is toggle on macOS, but not on windows
        elif key == arcade.key.M:
            self.maximize()

        # Toggle between Fullscreen and window
        #   keeps viewport coordinates the same   STRETCHED (FULLSCREEN)
        #   Instead of a one-to-one mapping to screen size, we use stretch/squash window to match the constants.
        #   src: http://arcade.academy/examples/full_screen_example.html
        elif key == arcade.key.F:
            self.update_current_screen()
            self._set_full_screen(not self.fullscreen)

    # toggle screen for fullscreen
    # BUG: doesn't work on macOS => see explanation in set_screen_to_display_simulator_at_startup() method
    def toggle_screen_used_for_fullscreen(self):

        # toggle only between screen 0 and 1 (other screens are ignored)
        self.current_screen_index = (self.current_screen_index + 1) % 2

        # override hidden screen parameter in window
        screens = get_screens()
        self._screen = screens[self.current_screen_index]

    def update_current_screen(self):
        """ using the windows position and size we determine on which screen it is currently displayed and make that
            current screen for displaying in fullscreen!!
        """

        screens = get_screens()
        if len(screens) == 1:
            return

        location = self.get_location()
        top_left_x = location[0]
        top_left_y = location[1]
        size = self.get_size()
        win_width = size[0]
        win_height = size[1]

        done = False
        locations = [location, (top_left_x + win_width, top_left_y), (top_left_x, top_left_y + win_height),
                     (top_left_x + win_width, top_left_y + win_height)]
        for location in locations:
            if done:
                break
            loc_x = location[0]
            loc_y = location[1]
            num = 0
            for screen in screens:
                within_screen_width = screen.x <= loc_x < (screen.x + screen.width)
                within_screen_height = screen.y <= (loc_y < (screen.y + screen.height))
                if within_screen_width and within_screen_height:
                    self.current_screen_index = num
                    done = True
                    break
                num += 1

        # override hidden screen parameter in window
        self._screen = screens[self.current_screen_index]

    def _set_full_screen(self, is_full_screen: bool = True):
        self.set_fullscreen(is_full_screen)

        # Instead of a one-to-one mapping, stretch/squash window to match the
        # constants. This does NOT respect aspect ratio. You'd need to
        # do a bit of math for that.
        self.set_viewport(0, self.screen_width, 0, self.screen_height)

        # HACK for macOS: without this hack fullscreen on the second screen is shifted downwards in the y direction
        #     By also calling the maximize function te position the fullscreen in second screen is corrected!)
        import platform
        if platform.system().lower() == "darwin":
            self.maximize()

    def on_resize(self, width, height):
        """ This method is automatically called when the window is resized. """

        # Call the parent. Failing to do this will mess up the coordinates, and default to 0,0 at the center and the
        # edges being -1 to 1.
        super().on_resize(self.screen_width, self.screen_height)

    def on_draw(self):
        """
        Render the simulation.
        """

        arcade.start_render()

        for obstacleList in self.world_state.static_obstacles:
            for shape in obstacleList.get_shapes():
                if shape.line_width == 1:
                    shape.draw()
                else:
                    print(shape)
        self.world_state.sprite_list.draw()

        for robot in self.world_state.get_robots():
            robot.get_sprites().draw()

            if debug:
                for sprite in robot.get_sprites():
                    sprite.draw_hit_box(color=RED, line_thickness=5)
                    if robot.debug_shapes is not None:
                        for shape in robot.debug_shapes:
                            shape.draw()
                    robot.debug_shapes.clear()

            if robot.is_stuck and self.msg_counter <= 0:
                self.msg_counter = self.frames_per_second * 3

        for robot in self.world_state.get_robots():
            self.sidebar.add_robot_info(robot.name, robot.values, robot.sounds)

        self.sidebar.draw()
        if self.msg_counter > 0:
            self.msg_counter -= 1

            arcade.draw_text(self.falling_msg, self.msg_x, self.screen_height - 100, arcade.color.RADICAL_RED,
                             14,
                             anchor_x="center")
            arcade.draw_text(self.restart_msg, self.msg_x, self.screen_height - 130, arcade.color.RADICAL_RED,
                             14,
                             anchor_x="center")

    def update(self, delta_time):
        """
        All the logic to move the robot. Collision detection is also performed.
        Callback to WorldSimulator.update is called
        """
        self.update_callback()

    def on_mouse_press(self, x: float, y: float, button: int, modifiers: int):
        self.world_state.set_object_at_position_as_selected((x, y))

    def on_mouse_release(self, x: float, y: float, button: int,
                         modifiers: int):
        self.world_state.unselect_object()

    def on_mouse_drag(self, x: float, y: float, dx: float, dy: float, buttons: int, modifiers: int):
        if buttons == arcade.MOUSE_BUTTON_LEFT:
            self.world_state.move_selected_object(dx, dy)
        if buttons == arcade.MOUSE_BUTTON_RIGHT:
            self.world_state.rotate_selected_object(dy)


from tqdm import tqdm
import pandas as pd
import numpy as np
import sys
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedKFold
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
import random as rnd
from imblearn.over_sampling import SMOTE
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix, precision_recall_curve, auc, roc_auc_score, roc_curve, recall_score, classification_report
import sor_wc_wk_joint_noredundant as sor
import time
import cv_split
import pickle

scaler = StandardScaler()

pca = PCA(0.99)

pfile = 'test_data/test_' + sys.argv[1] + '.npz'

params = np.load(pfile)

print(params.files)

train_classes = params['train_classes']
test_classes = params['test_classes']
train_index = params['train_index']
test_index = params['test_index']

risk_class_files = ['Drought.csv', 'Earthquakes.csv', 'Explosions.csv', 'Floods.csv', 'Forest_and_Brush_Fire.csv', 'Hazardous_and_Toxic_Substance.csv', 'Landslides.csv', 'Lighting.csv', 'Snowstorms.csv', 'Tornado.csv', 'Tropical Storms.csv', 'Volcanoes.csv', 'Water_Pollution.csv']

risk_class_dict = {}
for i in range(len(risk_class_files)):
  risk_class_dict[i+1] = risk_class_files[i]

def remove_label(docs):
  for i in range(len(docs)):
    docs[i] = docs[i].replace('"1, ','').replace('"0, ','').replace("'0, ",'').replace("'0, ",'')
  return docs

risk_classes = {}
for risk_file in risk_class_files:
  risk_classes[risk_file] = pd.read_csv('../data/NYTimes_data/'+risk_file, header = None)[0].tolist()

non_risk_file = 'non_risk_docs.csv'

non_risk_class = pd.read_csv('../data/NYTimes_data/'+non_risk_file, header = None)[0].tolist()

X = []
Y = []

class_id = 1

for risk_file in risk_class_files:
  X += risk_classes[risk_file]
  Y += [class_id] * len(risk_classes[risk_file])
  class_id += 1

X += non_risk_class
Y += [0] * len(non_risk_class)

X = remove_label(X)

tfidf = TfidfVectorizer(ngram_range=(1,1), stop_words='english', token_pattern=u'(?ui)\\b\\w*[a-z]+\\w*\\b')

features = tfidf.fit_transform(X).toarray()
labels = Y

def run_test(features, labels, train_classes, test_classes, train_index, test_index):
    features = np.array(features)
    labels = np.array(labels)
    xtrain = features[np.isin(labels,train_classes),:]
    ytrain = labels[np.isin(labels,train_classes)]

    xtest = features[np.isin(labels,test_classes),:]
    ytest = labels[np.isin(labels,test_classes)]

    RR = 0
    R2Ri = 0
    RNR = 0
    NRNR = 0
    NRR = 0
    RiR = np.zeros(len(test_classes))
    RiNR = np.zeros(len(test_classes))

    X_train, X_test = xtrain[train_index], xtrain[test_index]
    y_train, y_test = ytrain[train_index], ytrain[test_index]

    scaler.fit(X_train)
    pca.fit(scaler.transform(X_train))
    X_train = pca.transform(scaler.transform(X_train))
    X_test = pca.transform(scaler.transform(X_test))
    xtest = pca.transform(scaler.transform(xtest))

    y_test_l = y_test.tolist()

    model = sor.HierarchicalClassifierModel(input_size = X_train[0].size, num_classes = len(risk_class_files), learning_rate = 1e-3, num_epochs = 1000, batch_size = 100, l1 = 0, l2 = 0, train_classes = train_classes)

    parameters = {'l1':np.logspace(-2,2,5), 'l2':np.append(np.logspace(-3,1,5),0)}
    splitter = cv_split.UnseenTestSplit()
    cmodel = GridSearchCV(model, parameters, cv = splitter, verbose=5, n_jobs = 10, error_score='raise')
    cmodel.fit(X_train, y_train.ravel())

    print('best params: l1=', cmodel.best_params_['l1'], 'l2=', cmodel.best_params_['l2'])

    model = sor.HierarchicalClassifierModel(input_size = X_train[0].size, num_classes = len(risk_class_files), learning_rate = 1e-3, num_epochs = 1000, batch_size = 100, l1 = cmodel.best_params_['l1'], l2 = cmodel.best_params_['l2'], train_classes = train_classes)

    model.fit(X_train, y_train)

    y_pred = model.predict(X_test, 0)
    y_pred_score = model.predict_score(X_test, 0)
    np.savetxt('test_results/' + sys.argv[1] + '_pca_joint_seen.out', y_pred_score)

    for j in range(len(y_test)):
            if y_test_l[j] >= 1:
                if y_pred[j] == 1:
                    RR += 1
                    y_pred_class = model.predict(X_test[j,:].reshape(1,-1), int(y_test[j]))
                    if y_pred_class == 1:
                      R2Ri += 1
                else:
                    RNR += 1
            else:
                if y_pred[j] == -1:
                    NRNR += 1
                else:
                    NRR += 1

    for classk in range(len(test_classes)):
            xtest_ri = xtest[ytest == test_classes[classk]]
            y_pred_ri = model.predict(xtest_ri, 0)
            y_pred_ri_score = model.predict_score(xtest_ri, 0)
            np.savetxt('test_results/' + sys.argv[1] + '_pca_joint_unseen_' + str(classk)+ '.out', y_pred_ri_score)

            for j in range(len(y_pred_ri)):
              if y_pred_ri[j] == 1:
                RiR[classk] += 1
              else:
                RiNR[classk] += 1

    print(RR, RNR, NRR, NRNR, RiR, RiNR, R2Ri)
    pickle.dump(model, open('test_results/trained_model_' + sys.argv[1] + '_pca_joint' + '.m', 'wb'))

print('Train classes:', train_classes)
print('Test classes:', test_classes)

run_test(features, labels, train_classes, test_classes, train_index, test_index)


# -*- coding: utf-8 -*-
# @Time    : 2018/5/21 9:07
# @Author  : lanxin yang
# @FileName: augmentation.py

import cv2
import numpy as np
from utils.config import *

def gamma_transform(img,gamma_vari):
    """
    Gamma变换： y=x^gamma;
    通过非线性变换让图像从对曝光强度的线性响应变得更接近人眼感受到的响应
    gamma>1, 拉伸图像中灰度级较低的区域，同时压缩灰度级较高的部分，图像整体变暗；
    gamma<1, 拉伸图像中灰度级较高的区域，同时压缩灰度级较低的部分，图像整体变亮；
    """
    log_gamma_vari = np.log(gamma_vari)
    alpha = np.random.uniform(-log_gamma_vari, log_gamma_vari)
    gamma = np.exp(alpha)

    gamma_table=[np.power(x/255.0,gamma)*255.0 for x in range(256)]
    gamma_table=np.round(np.array(gamma_table)).astype(np.uint8)

    return cv2.LUT(img,gamma_table)

def rotate(img,lab,angle):
    """
    旋转操作
    """
    M_rotate = cv2.getRotationMatrix2D((IMG_W/2, IMG_H/2), angle, 1)
    img = cv2.warpAffine(img, M_rotate, (IMG_W, IMG_H))
    lab = cv2.warpAffine(lab, M_rotate, (IMG_W, IMG_H))
    return img,lab

def blur(img):
    """
    用低通滤波来平滑图像
    """
    img=cv2.blur(img,(3,3))
    return img

def add_noise(img):
    """
    加噪操作
    """
    for i in range(200):
        temp_x = np.random.randint(0,img.shape[0])
        temp_y = np.random.randint(0,img.shape[1])
        img[temp_x][temp_y] = 255
    return img

def data_augment(img, lab):
    """
    增强操作
    """
    if np.random.random() < 0.25:
        img, yb = rotate(img, lab, 90)
    if np.random.random() < 0.25:
        img, yb = rotate(img, lab, 180)
    if np.random.random() < 0.25:
        img, yb = rotate(img, lab, 270)
    if np.random.random() < 0.25:
        img = cv2.flip(img, 1)  # flipcode > 0：沿y轴翻转
        lab = cv2.flip(lab, 1)

    if np.random.random() < 0.25:
        img = gamma_transform(img, 1.0)

    if np.random.random() < 0.25:
        img = blur(img)

    if np.random.random() < 0.2:
        img = add_noise(img)

    return img, lab


#!/usr/bin/python3
import matplotlib
matplotlib.use('Agg')
import os
import sys
import pylab

import haitai.graph
import haitai.common
import haitai

if __name__ == '__main__':
    ndays=3000
    bk='全部'
    bk='中小'
    bk='主板'

    # dates
    dates=haitai.common.recent_n_days(ndays)

    # stocks
    th = set(haitai.get_symbol_list(bk))

    d = 'data/163_daily'
    fs = os.listdir(d)
    fs = [f for f in fs if f != '000300.ss']
    print(fs)
    #th = set(list(th)[:100]) # for debug only

    # get data
    pss, vols = haitai.common.load_stock_set(fs,ndays,dates)

    # compute
    p=sum(pss)
    print(p)
    p=p/p[0]
    v=haitai.common.mean_with_nan(vols)

    # fig
    pylab.figure(figsize=(12,6))
    both_ticks=haitai.graph.gen_ticks(dates)

    ax=pylab.subplot(2,1,1)
    haitai.graph.draw_lines(ax,[[p]],log=True)
    haitai.graph.draw_grid(ax,both_ticks)

    ax=pylab.subplot(2,1,2)
    haitai.graph.draw_lines(ax,[[v,'k']],log=True)
    haitai.graph.draw_grid(ax,both_ticks)

    filename=haitai.output_dir+'/'+bk+'_mean.svg'
    pylab.savefig(filename)
    pylab.clf()

    # save mean curve
    file=open(bk+'_mean.txt','w')
    for dd,pp,vv in zip(dates,p,v):
        print(dd,pp,vv,file=file)
    file.close()