# Install python 3, duh!
# Run the command below in a cmd window to install the needed packages, without the #, duh!
# pip install bs4 requests pandas openpyxl lxml html5lib
# Run the python file with the included batch file, DUH!

try:
    # Error handling if something happens during script initialisation
    from csv import QUOTE_ALL  # Needed to export data to CSV
    from bs4 import BeautifulSoup  # Needed to parse the dynamic webpage of the Ducanator
    from requests import get  # Needed to get the webpage of the Ducanator
    from re import search  # Needed to find the json string to import into pandas
    from pandas import read_csv, set_option, concat, DataFrame, read_json, read_html, ExcelWriter  # Needed to convert the json string into a usable dataframe object for manipulation
    from traceback import format_exc  # Needed for more friendly error messages.
    from openpyxl import load_workbook
    from numpy import arange
    from os import path
except ModuleNotFoundError:
    print('OOPSIE WOOPSIE!! Uwu We made a fucky wucky!! A wittle fucko boingo! The code monkeys at our headquarters are working VEWY HAWD to fix this!')
    print('You didn\'t install the packages like I told you to. Please run \"pip install bs4 requests pandas\" in a cmd window to install the required packages!')
    print('\033[1;31m' + format_exc())
    exit(1)

try:
    #User Variables
    workbook_name = 'Prime_Relic_Data.xlsx'
    csv_name = 'Prime-Relic Data.csv'
    sheet_name_day = 'Day'
    sheet_name_hour = 'Hour'
    sheet_name_relic = 'Relic_Data'
    retry_attempts = 10
    # Sets the URL to scrape, because hard-coding is bad
    print('Downloading Ducat Data')
    url_ducats = "https://warframe.market/tools/ducats"
    # Scrapes the given URL
    soup = str(BeautifulSoup(get(url_ducats).content, "html.parser")).replace('\n', '')
    print('Ducat Data Downloaded')
    print('Processing Ducat Data')
    # Finds the needed json string for item data, previous hour data, and previous day data.
    # Slices off the first bit to make a valid json string for pandas later
    items = search('"items": (\[(?:\[??[^\[]*?\]))', soup).group(0)[9:]
    previous_hour = search('"previous_hour": (\[(?:\[??[^\[]*?\]))', soup).group(0)[17:]
    previous_day = search('"previous_day": (\[(?:\[??[^\[]*?\]))', soup).group(0)[16:]

    # Reads and sanitises the item data into a pandas dataframe
    df_items = read_json(items)
    df_items = df_items.drop(columns=['url_name', 'thumb'])
    df_items = df_items.reindex(columns=['id', 'item_name'])

    # Reads and sanitises the previous day data into a pandas dataframe
    df_previous_day = read_json(previous_day)
    df_previous_day = df_previous_day.drop(columns=['id', 'plat_worth', 'median'])
    df_previous_day = df_previous_day.rename(columns={'item': 'id'})
    # Merges the item data and previous day data on the id column, drops the redundant id column, then renames the column names for export
    df_previous_day_merged = df_items.merge(df_previous_day, how='inner', on='id')
    df_previous_day_merged = df_previous_day_merged.drop(columns=['id'])
    df_previous_day_merged = df_previous_day_merged.reindex(columns=['item_name', 'datetime', 'ducats_per_platinum', 'ducats', 'wa_price','ducats_per_platinum_wa', 'position_change_month', 'position_change_week', 'position_change_day', 'volume'])
    df_previous_day_merged = df_previous_day_merged.sort_values(by='item_name')
    df_previous_day_merged['datetime'] = df_previous_day_merged['datetime'].astype(str).str[:-6]

    # Reads and sanitises the previous hour data into a pandas dataframe
    df_previous_hour = read_json(previous_hour)
    df_previous_hour = df_previous_hour.drop(columns=['id', 'plat_worth', 'median'])
    df_previous_hour = df_previous_hour.rename(columns={'item': 'id'})
    # Merges the item data and previous hour data on the id column, drops the redundant id column, then renames the column names for export
    df_previous_hour_merged = df_items.merge(df_previous_hour, how='inner', on='id')
    df_previous_hour_merged = df_previous_hour_merged.drop(columns=['id'])
    df_previous_hour_merged = df_previous_hour_merged.reindex(columns=['item_name', 'datetime', 'ducats_per_platinum', 'ducats', 'wa_price','ducats_per_platinum_wa', 'position_change_month', 'position_change_week', 'position_change_day', 'volume'])
    df_previous_hour_merged = df_previous_hour_merged.sort_values(by='item_name')
    df_previous_hour_merged['datetime'] = df_previous_hour_merged['datetime'].astype(str).str[:-6]
    df_previous_hour_merged = df_previous_hour_merged.reset_index(drop=True)

    print('Ducat Data Processed')
    # Fuck Comments
    print('Downloading Relic Data')
    url_relics = "https://n8k6e2y6.ssl.hwcdn.net/repos/hnfvc0o3jnfvc873njb03enrf56.html"
    relic_data_txt_name = 'RelicData.txt'

    if path.isfile(relic_data_txt_name):
        with open(relic_data_txt_name) as f:
            soup = str(f.readlines())
        print("Loaded Local Relic Data")
    else:
        print("Loading Remote Item Data")

        for x in range(0, retry_attempts):
            try:
                soup = str(BeautifulSoup(get(url_relics).content, "html.parser")).replace('\n', '')
                print('Saving Local Data')
                with open(relic_data_txt_name, 'w') as f:
                    f.write(soup)
                break
            except Exception:
                print('Relic data download failed, retrying... ' + str(retry_attempts - x - 1) + ' attempts left...', end='\r')

    print('Relic Data Downloaded')
    print('Processing Relic Data')
    parsed_relics = search('
Relics:

.*?
', soup).group(0)[34:].replace('th>', 'td>').replace(r'', r'').replace('X Kuva', 'x Kuva')
    df_parsed_relics = read_html(parsed_relics, header=None)
    df_parsed_relics = df_parsed_relics[0].replace(to_replace=r'.+\((.+)\%\)', value=r'\1', regex=True)
    df_parsed_relics[1] = df_parsed_relics[1].astype(float)
    df_parsed_relics = df_parsed_relics.dropna(how='all').fillna(999)
    groups = df_parsed_relics.groupby(arange(len(df_parsed_relics.index)) // 7, sort=False).apply(lambda x: x.sort_values(by=1, ascending=False))
    groups[1] = ' (' + groups[1].astype(str) + '%)'
    groups = groups[0] + groups[1]
    groups = groups.replace(to_replace=r'\(999.0\%\)', value=r'', regex=True)
    templist = []
    templist2 = []
    for count, value in enumerate(groups):
        if count % 7 == 0 and count != 0:
            templist2.append(templist)
            templist = []
        templist.append(value)
    df_even_more_parsed_relics = DataFrame(templist2, columns=['Relic_Name', 'C1', 'C2', 'C3', 'U1', 'U2', 'Rare'])
    df_relic_class = df_even_more_parsed_relics['Relic_Name'].str.split().str[0]
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'Class', df_relic_class, allow_duplicates=True)
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'Type', df_even_more_parsed_relics['Relic_Name'].str.upper().str.split().str[1], allow_duplicates=True)
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'Refinement', df_even_more_parsed_relics['Relic_Name'].str.split().str[3].replace(to_replace=r'[\(\)]', value=r'', regex=True), allow_duplicates=True)
    dict = {'Exceptional':'','Flawless':'','Radiant':''}
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'C1_Raw', df_even_more_parsed_relics['C1'].replace(to_replace=r' \(.+\)',value='',regex=True))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'C2_Raw', df_even_more_parsed_relics['C2'].replace(to_replace=r' \(.+\)',value='',regex=True))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'C3_Raw', df_even_more_parsed_relics['C3'].replace(to_replace=r' \(.+\)',value='',regex=True))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'U1_Raw', df_even_more_parsed_relics['U1'].replace(to_replace=r' \(.+\)',value='',regex=True))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'U2_Raw', df_even_more_parsed_relics['U2'].replace(to_replace=r' \(.+\)',value='',regex=True))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'Rare_Raw', df_even_more_parsed_relics['Rare'].replace(to_replace=r' \(.+\)',value='',regex=True))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'C1_Odds', df_even_more_parsed_relics['C1'].replace(to_replace=r'.+\((.+)\%\)',value=r'\1',regex=True).astype(float))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'C2_Odds', df_even_more_parsed_relics['C2'].replace(to_replace=r'.+\((.+)\%\)',value=r'\1',regex=True).astype(float))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'C3_Odds', df_even_more_parsed_relics['C3'].replace(to_replace=r'.+\((.+)\%\)',value=r'\1',regex=True).astype(float))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'U1_Odds', df_even_more_parsed_relics['U1'].replace(to_replace=r'.+\((.+)\%\)',value=r'\1',regex=True).astype(float))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'U2_Odds', df_even_more_parsed_relics['U2'].replace(to_replace=r'.+\((.+)\%\)',value=r'\1',regex=True).astype(float))
    df_even_more_parsed_relics.insert(len(df_even_more_parsed_relics.columns), 'Rare_Odds', df_even_more_parsed_relics['Rare'].replace(to_replace=r'.+\((.+)\%\)',value=r'\1',regex=True).astype(float))
    df_even_more_parsed_relics = df_even_more_parsed_relics.replace(to_replace=r'Systems Blueprint',value=r'Systems', regex=True)
    df_even_more_parsed_relics = df_even_more_parsed_relics.replace(to_replace=r'Neuroptics Blueprint',value=r'Neuroptics', regex=True)
    df_even_more_parsed_relics = df_even_more_parsed_relics.replace(to_replace=r'Chassis Blueprint',value=r'Chassis', regex=True)
    #print(df_even_more_parsed_relics.head(5))
    #df_even_more_parsed_relics['Relic_Name'] = df_even_more_parsed_relics['Relic_Name'].str.split(n=1).str[1]
    #df_axi = df_even_more_parsed_relics[df_even_more_parsed_relics['Relic_Class']=='Axi'].reset_index(drop=True)
    #df_lith = df_even_more_parsed_relics[df_even_more_parsed_relics['Relic_Class']=='Lith'].reset_index(drop=True)
    #df_meso = df_even_more_parsed_relics[df_even_more_parsed_relics['Relic_Class']=='Meso'].reset_index(drop=True)
    #df_neo = df_even_more_parsed_relics[df_even_more_parsed_relics['Relic_Class']=='Neo'].reset_index(drop=True)
    #df_requiem = df_even_more_parsed_relics[df_even_more_parsed_relics['Relic_Class']=='Requiem'].reset_index(drop=True)
    #df_final_export_relic = concat([df_axi,df_lith,df_meso,df_neo,df_requiem], axis=1, ignore_index=True)
    #print(df_even_more_parsed_relics)
    print('Relic Data Processed')

    # Export data
    print('Exporting Worksheet')
    df_even_more_parsed_relics.to_csv(csv_name, index=None, quoting=QUOTE_ALL)
    df_previous_day_merged.to_csv('DayPrices.csv', index=None, quoting=QUOTE_ALL)
    with ExcelWriter(workbook_name, mode='a', engine='openpyxl', if_sheet_exists='replace') as writer:
        df_previous_day_merged.to_excel(writer, sheet_name=sheet_name_day)
        df_previous_hour_merged.to_excel(writer, sheet_name=sheet_name_hour)
        df_even_more_parsed_relics.to_excel(writer, sheet_name=sheet_name_relic)
        #df_final_export_relic.to_excel(writer, sheet_name=sheet_name_relic)
    book = load_workbook(workbook_name)
    sheet = book[sheet_name_day]
    sheet.delete_cols(1,1)
    sheet = book[sheet_name_hour]
    sheet.delete_cols(1,1)
    sheet = book[sheet_name_relic]
    sheet.delete_cols(1,1)
    book.save(workbook_name)
    print('If you see this message, things should have worked correctly. Remove the \"pause\" from the batch script to automatically close this window after use.')

except Exception:
    # Error handling if something happens during the main script
    print('OOPSIE WOOPSIE!! Uwu We made a fucky wucky!! A wittle fucko boingo! The code monkeys at our headquarters are working VEWY HAWD to fix this!')
    print('\033[1;31m' + format_exc())
    exit(1)


from .models import User
from django import forms

class UserForm(forms.ModelForm):
     class Meta:
         # specify model to be used
         model = User

         # specify fields to be used
         fields = [
             "first_name",
             "second_name",
         ]


"""
Simple implementation of (Fisher's) Linear Discriminant Analysis.
Thanks to: https://www.python-course.eu/linear_discriminant_analysis.php

The L. D. Matrix is a transformation matrix which best separates
the instances of different classes in data projection.
"""
import numpy as np

class LDA:
    def fit(self, X, y):
        """Fit dataset into LDA model."""
        self.X = np.array(X)
        self.y = np.array(y)

        self.classes, self.cls_freqs = np.unique(y, return_counts=True)

    def _scatter_within(self):
        """This measure describes how scattered are each class."""
        scatter_within = np.array([
            (cls_freq - 1) * np.cov(self.X[self.y == cls, :], rowvar=False)
            for cls, cls_freq in zip(self.classes, self.cls_freqs)
        ]).sum(axis=0)

        return scatter_within

    def _scatter_between(self):
        """This measure describes the separation between different classes."""
        class_means = np.array(
            [self.X[self.y == cls, :].mean(axis=0) for cls in self.classes])

        total_mean = self.X.mean(axis=0)

        scatter_factor = class_means - total_mean

        scatter_between = np.array([
            freq * np.outer(sf, sf)
            for freq, sf in zip(self.cls_freqs, scatter_factor)
        ]).sum(axis=0)

        return scatter_between

    def _get_eig(self, sw, sb):
        """Get eigenval/vec from (ScatterWithin)^(-1)*(ScatterBetween) mat."""
        sw_inv = np.linalg.inv(sw)

        return np.linalg.eig(np.matmul(sw_inv, sb))

    def _project(self, eig, num_dim):
        """Get the K (``num_dim``) most expressive eigenvalues/vectors."""
        eig_vals, eig_vecs = eig

        eig_vals, eig_vecs = zip(
            *sorted(
                zip(eig_vals, eig_vecs),
                key=lambda item: item[0],
                reverse=True)[:num_dim])

        return eig_vals, eig_vecs

    def predict(self, max_dim=2):
        """Create transf. matrix which best separates the fitted data proj."""
        sw = self._scatter_within()
        sb = self._scatter_between()

        max_dim = min(max_dim, self.classes.size-1)

        eig = self._get_eig(sw, sb)

        eig_vals, eig_vecs = self._project(eig, num_dim=max_dim)

        _, num_col = self.X.shape

        self.eig_vals = np.array(eig_vals)
        self.transf_mat = np.concatenate(eig_vecs).reshape(num_col, max_dim)

        self.transf_mat = self.transf_mat.real

        return self.transf_mat

    def wilks_lambda(self):
        """Compute Wilks' Lambda measure using eigenvalues of L. D. matrix."""
        return np.prod(1.0 / (1.0 + self.eig_vals))

    def canonical_corr(self):
        """Calculate canonical correlation values from L. D. matrix."""
        return (self.eig_vals / (1.0 + self.eig_vals))**0.5

if __name__ == "__main__":
    from sklearn import datasets
    iris = datasets.load_iris()

    model = LDA()
    model.fit(iris.data, iris.target)
    ans = model.predict(max_dim=2)

    print("Transformation Matrix:", ans, sep="\n", end="\n\n")
    print("Eigenvalues of L. D. matrix:", model.eig_vals, end="\n\n")
    print("Canonical Correlation:", model.canonical_corr(), end="\n\n")
    print("Wilks' Lambda:", model.wilks_lambda())


import os
import time

from telegram import Update
from telegram.ext import Updater, CommandHandler

from data.copart import Copart
from data.utils import EditThread

def search(update: Update, context):
    default_args = [*([0, float('inf')] * 2)]
    names = ['year_from', 'year_to', 'price_from', 'price_to']
    try:
        filters = {names[i]: int(context.args[i]) if i < len(context.args) and context.args[i] != '-1'
                   else default_args[i] for i in range(len(default_args))}
    except ValueError:
        return update.message.reply_text('Все аргументы должны быть целочисленными!')
    message = update.message.reply_text('Начинаем поиск...')
    edit_thread = EditThread(message)
    edit_thread.start()
    start_time = time.time()
    copart = Copart()
    output = copart.get_data(filters)
    edit_thread.stop()
    update.message.reply_text(f'Найдено {len(output)} автомобилей.\n'
                              f'Время поиска: {time.time() - start_time:.2f} секунд')
    for car in output:
        update.message.reply_text(car['ld'])

def start(update, _):
    update.message.reply_text('Привет! Это бот-парсер американских автобирж. '
                              'Введите /search {year_from} {year_to} {price_from} {price_to} '
                              'для поиска авто')

def main():
    updater = Updater(os.getenv('tg_token'))
    updater.dispatcher.add_handler(CommandHandler('start', start))
    updater.dispatcher.add_handler(CommandHandler('search', search, pass_args=True))
    updater.start_polling()
    updater.idle()

if __name__ == '__main__':
    main()


import json
import tweepy
from tweepy import OAuthHandler

class _Twitter(type):
	def __call__(cls, *args, **kwargs):
		if not hasattr(cls, 'instance'):
			cls.instance = super(_Twitter, cls).__call__(*args, **kwargs)
		return cls.instance

class Twitter(object, metaclass=_Twitter):
	def __init__(self):
		self.consumer_key = ''
		self.consumer_secret = ''
		self.access_key = ''
		self.access_secret = ''
		self.session = None
		self.api = None
		self.maxPage = 0
		self.maxCount = 0

	def loadConfig(self):
		with open("config.json") as data:
			conf = json.load(data)
			self.consumer_key = conf['consumer_key']
			self.consumer_secret = conf['consumer_secret']
			self.access_key = conf['access_key']
			self.access_secret = conf['access_secret']
			self.maxCount = int(conf['maxCount'])
			self.maxPage = int(conf['maxPage'])
		return self

	def auth(self):
		if self.session is None:
			self.session = OAuthHandler(self.consumer_key, self.consumer_secret)
			self.session.set_access_token(self.access_key, self.access_secret)
			self.api = tweepy.API(self.session, wait_on_rate_limit=True, wait_on_rate_limit_notify=True, compression=True)
		return self

	def getAPI(self):
		return self.api

	def getMaxCount(self):
		return self.maxCount

	def getMaxPage(self):
		return self.maxPage

API = Twitter().loadConfig().auth().getAPI()
MaxCount = Twitter().getMaxCount()
MaxPage = Twitter().getMaxPage()


from datetime import datetime, timezone

from app.configs.database import SingletonSQLAlchemy

db = SingletonSQLAlchemy()

class BaseModel(db.Model):
    __abstract__ = True

    id = db.Column(db.Integer, primary_key=True)
    create_at = db.Column(db.DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
    updated_at = db.Column(db.DateTime(timezone=True), nullable=True)

    def before_save(self, *args, **kwargs):
        return

    def before_save(self, *args, **kwargs):
        return

    def save(self, commit=True):
        self.before_save()

        db.session.add(self)
        if commit:
            try:
                db.session.commit()
            except Exception as error:
                db.session.rollback()
                raise error

        self.before_save()

    def delete(self, commit=True):
        db.session.delete(self)
        if commit:
            db.session.delete(self)


from lxml import etree
import dateutil.parser
from pandas import DataFrame, Series
import pandas as pd
from functools import reduce
import json
import pymongo
from pymongo import MongoClient

class MroReader:
    def __init__(self, afilter, **kwargs):
        super().__init__(**kwargs)
        self.item_dicts=[]
        self.afilter=afilter

    def display(self):
        for item_dict in self.item_dicts:
            print(item_dict)

    def read(self, item_measurement, item_id):
        for item_element in item_measurement:
            if item_element.tag == 'smr':
                item_key = item_element.text.replace('MR.', '').split(' ')
            else:
                centerFilled=False
                item_dict = {}
                neighbor_list=[]
                for item_v in item_element:
                    item_value = item_v.text.replace('NIL', '-1').split(' ')
                    _item_sub_dict = dict(zip(item_key, map(int, item_value)))
                    _item_sub_dict = {k: v for k, v in _item_sub_dict.items() if not any(ext in k for ext in self.afilter)}
                    if _item_sub_dict['LteNcPci']>=0:
                        _neighbor={}
                        _neighbor.update({'Pci': _item_sub_dict['LteNcPci']})
                        _neighbor.update({'Rsrp': _item_sub_dict['LteNcRSRP']})
                        neighbor_list.append(_neighbor)
                    else:
                        break
                    if not centerFilled:
                        item_dict.update(item_element.attrib)
                        item_dict.update({'Rsrp': _item_sub_dict['LteScRSRP']})
                        item_dict.update({'SinrUl': _item_sub_dict['LteScSinrUL']})
                        item_dict.update({'Ta': _item_sub_dict['LteScTadv']})
                        item_dict.update({'Pci': _item_sub_dict['LteScPci']})
                        centerFilled=True
                if len(neighbor_list)>0:
                    item_dict.update({'NeighborList': neighbor_list})
                    self.item_dicts.append(item_dict)

    def read_zte(self, item_measurement, item_id):
        for item_element in item_measurement:
            if item_element.tag == 'smr':
                item_key = item_element.text.replace('MR.', '').split(' ')
                if 'LteScEarfcn' not in item_key:
                    return
            else:
                centerFilled=False
                item_dict = {}
                neighbor_list=[]
                for item_v in item_element:
                    item_value = item_v.text.replace('NIL', '-1').split(' ')
                    _item_sub_dict = dict(zip(item_key, map(int, item_value)))
                    _item_sub_dict = {k: v for k, v in _item_sub_dict.items() if not any(ext in k for ext in self.afilter)}
                    if _item_sub_dict['LteNcPci']>=0:
                        _neighbor={}
                        _neighbor.update({'Pci': _item_sub_dict['LteNcPci']})
                        _neighbor.update({'Rsrp': _item_sub_dict['LteNcRSRP']})
                        neighbor_list.append(_neighbor)
                    else:
                        break
                    if not centerFilled:
                        item_dict.update({'id': item_id+'-'+item_element.attrib['MR.objectId']})
                        item_dict.update({'Rsrp': _item_sub_dict['LteScRSRP']})
                        item_dict.update({'SinrUl': _item_sub_dict['LteScSinrUL']})
                        item_dict.update({'Ta': _item_sub_dict['LteScTadv']})
                        item_dict.update({'Pci': _item_sub_dict['LteScPci']})
                        centerFilled=True
                if len(neighbor_list)>0:
                    item_dict.update({'NeighborList': neighbor_list})
                    self.item_dicts.append(item_dict)

    def _filter_by_neighbor_len(self, length):
        return list(filter(lambda x: True if len(x['NeighborList'])==length else False, self.item_dicts))

    def _map_neighbor_rsrp_diff(self, index):
        measureList=self._filter_by_neighbor_len(index)
        if len(measureList)==0:
            return []
        return list(map(lambda item: {
            'CellId': item['id'],
            'NeighborPci': item['NeighborList'][index-1]['Pci'],
            'RsrpDiff': item['Rsrp']-item['NeighborList'][index-1]['Rsrp'],
            'Rsrp': item['Rsrp'],
            'Pci': item['Pci'],
            'Ta': item['Ta'],
            'SinrUl': item['SinrUl']
        }, measureList))

    def map_rsrp_diff(self):
        diff_list=list(map(lambda index: self._map_neighbor_rsrp_diff(index+1), list(range(6))))
        combined_list=reduce(lambda first,second: first+second,diff_list,[])
        if len(combined_list)==0:
            return []
        stat_list=list(map(lambda item: {
            'CellId': item['CellId'],
            'NeighborPci': item['NeighborPci'],
            'Pci': item['Pci'],
            'Diff0': 1 if item['RsrpDiff']<=0 else 0,
            'Diff3': 1 if item['RsrpDiff']<=3 and item['RsrpDiff']>0 else 0,
            'Diff6': 1 if item['RsrpDiff']<=6 and item['RsrpDiff']>3 else 0,
            'Diff9': 1 if item['RsrpDiff']<=9 and item['RsrpDiff']>6 else 0,
            'Diff12': 1 if item['RsrpDiff']<=12 and item['RsrpDiff']>9 else 0,
            'DiffLarge': 1 if item['RsrpDiff']>12 else 0,
            'RsrpBelow120': 1 if item['Rsrp']<20 else 0,
            'RsrpBetween120110': 1 if item['Rsrp']<30 and item['Rsrp']>=20 else 0,
            'RsrpBetween110105': 1 if item['Rsrp']<35 and item['Rsrp']>=30 else 0,
            'RsrpBetween105100': 1 if item['Rsrp']<40 and item['Rsrp']>=35 else 0,
            'RsrpBetween10090': 1 if item['Rsrp']<50 and item['Rsrp']>=40 else 0,
            'RsrpAbove90': 1 if item['Rsrp']>=50 else 0,
            'Ta0or1': 1 if item['Ta']==0 or item['Ta']==1 else 0,
            'Ta2or3': 1 if item['Ta']==2 or item['Ta']==3 else 0,
            'Ta4or5': 1 if item['Ta']==4 or item['Ta']==5 else 0,
            'Ta6or7': 1 if item['Ta']==6 or item['Ta']==7 else 0,
            'Ta8or9': 1 if item['Ta']==8 or item['Ta']==9 else 0,
            'Ta10to12': 1 if item['Ta']>=10 and item['Ta']<=12 else 0,
            'Ta13to15': 1 if item['Ta']>=13 and item['Ta']<=15 else 0,
            'Ta16to19': 1 if item['Ta']>=16 and item['Ta']<=19 else 0,
            'Ta20to24': 1 if item['Ta']>=20 and item['Ta']<=24 else 0,
            'Ta25to29': 1 if item['Ta']>=25 and item['Ta']<=29 else 0,
            'Ta30to39': 1 if item['Ta']>=30 and item['Ta']<=39 else 0,
            'TaAbove40': 1 if item['Ta']>=40 else 0,
            'SinrUl0to9': 1 if item['SinrUl']>=0 and item['SinrUl']<=9 else 0,
            'SinrUl10to19': 1 if item['SinrUl']>=10 and item['SinrUl']<=19 else 0,
            'SinrUl20to24': 1 if item['SinrUl']>=20 and item['SinrUl']<=24 else 0,
            'SinrUl25to29': 1 if item['SinrUl']>=25 and item['SinrUl']<=29 else 0,
            'SinrUl30to34': 1 if item['SinrUl']>=30 and item['SinrUl']<=34 else 0,
            'SinrUlAbove35': 1 if item['SinrUl']>=35 else 0
        }, combined_list))
        df = DataFrame(stat_list)
        stat=df.groupby(['CellId','Pci','NeighborPci']).sum().reset_index()
        return json.loads(stat.T.to_json()).values()

class MrsReader:
    def __init__(self, mrNames, startTime, date_dir, db, **kwargs):
        self.mrNames=mrNames
        self.startTime=startTime
        self.date_dir=date_dir
        self.db=db
        return super().__init__(**kwargs)

    def read(self, item_measurement):
        mrName=item_measurement.attrib['mrName'].replace('MR.','')
        if mrName in self.mrNames:
            item_dicts=[]
            for item_element in item_measurement.iterchildren():
                if item_element.tag == 'smr':
                    item_key = item_element.text.replace('MR.', '').replace('.','_').split(' ')
                else:
                    item_dict={}
                    item_dict.update({'CellId': item_element.attrib['id']})
                    item_value = item_element[0].text.split(' ')
                    item_dict.update(dict(zip(item_key, map(int, item_value))))
                    item_dict.update({'StartTime': self.startTime})
                    item_dicts.append(item_dict)
            if len(item_dicts)>0:
                self.db['mrs_'+mrName+'_'+self.date_dir].insert_many(item_dicts)

    def read_zte(self, item_measurement, eNodebId):
        mrName=item_measurement.attrib['mrName'].replace('MR.','')
        if mrName in self.mrNames:
            item_dicts=[]
            for item_element in item_measurement.iterchildren():
                if item_element.tag == 'smr':
                    item_key = item_element.text.replace('MR.', '').replace('.','_').split(' ')
                else:
                    item_dict={}
                    item_dict.update({'CellId': eNodebId+'-'+item_element.attrib['MR.objectId']})
                    item_value = item_element[0].text.split(' ')
                    item_dict.update(dict(zip(item_key, map(int, item_value))))
                    item_dict.update({'StartTime': self.startTime})
                    item_dicts.append(item_dict)
            if len(item_dicts)>0:
                self.db['mrs_'+mrName+'_'+self.date_dir].insert_many(item_dicts)


import re
import json
import pprint

from django.core.urlresolvers import reverse
from django.test import TestCase
from django.test.client import Client

from assessment.tests import build_assessments_for_permissions_testing
from utils.helper import HAWCDjangoJSONEncoder

from .models import SummaryText

class SummaryTextTests(TestCase):
    def setUp(self):
        build_assessments_for_permissions_testing(self)

    @staticmethod
    def clean_json(json_dump):
        remove_fields = ['created', 'last_updated', 'slug', 'text', 'assessment']
        for node in json_dump:
            node.pop('id')
            for field in remove_fields:
                node['data'].pop(field)
            if node.get('children'):
                SummaryTextTests.clean_json(node['children'])

    def test_adding_texts(self):
        lvl_1a = SummaryText.add_summarytext(assessment=self.assessment_working,
                                             title='lvl_1a',
                                             slug='lvl_1a',
                                             text='text')

        lvl_1b = SummaryText.add_summarytext(assessment=self.assessment_working,
                                             title='lvl_1b',
                                             slug='lvl_1b',
                                             text='text')

        lvl_2a = SummaryText.add_summarytext(assessment=self.assessment_working,
                                             parent=[lvl_1a],
                                             title='lvl_2a',
                                             slug='lvl_2a',
                                             text='text')

        lvl_2b = SummaryText.add_summarytext(assessment=self.assessment_working,
                                             sibling=[lvl_2a],
                                             title='lvl_2b',
                                             slug='lvl_2b',
                                             text='text')

        assessment_root = SummaryText.get_assessment_root_node(self.assessment_working)

        tree_form = SummaryText.dump_bulk(assessment_root)
        # print pprint.pprint(tree_form)

        SummaryTextTests.clean_json(tree_form)
        self.assertEqual(json.dumps(tree_form),"""[{"data": {"title": "assessment-1"}, "children": [{"data": {"title": "lvl_1a"}, "children": [{"data": {"title": "lvl_2a"}}, {"data": {"title": "lvl_2b"}}]}, {"data": {"title": "lvl_1b"}}]}]""")

        # Swap 2a and 2b
        lvl_2b.move_summarytext(parent=lvl_1a, sibling=None)
        tree_form = SummaryText.dump_bulk(assessment_root)
        SummaryTextTests.clean_json(tree_form)
        self.assertEqual(json.dumps(tree_form),"""[{"data": {"title": "assessment-1"}, "children": [{"data": {"title": "lvl_1a"}, "children": [{"data": {"title": "lvl_2b"}}, {"data": {"title": "lvl_2a"}}]}, {"data": {"title": "lvl_1b"}}]}]""")

        # Swap back
        lvl_2b.move_summarytext(parent=None, sibling=lvl_2a)
        tree_form = SummaryText.dump_bulk(assessment_root)
        SummaryTextTests.clean_json(tree_form)
        self.assertEqual(json.dumps(tree_form),"""[{"data": {"title": "assessment-1"}, "children": [{"data": {"title": "lvl_1a"}, "children": [{"data": {"title": "lvl_2a"}}, {"data": {"title": "lvl_2b"}}]}, {"data": {"title": "lvl_1b"}}]}]""")


# -*- coding: utf-8 -*-
"""
Created on Fri Mar 22 18:33:47 2019

@author: user
"""
import numpy as np
import struct
import matplotlib.pyplot as plt #for displaying data
from mpl_toolkits.mplot3d import Axes3D
import csv
import pandas as pd #to manage dataframe
import struct
from math import sqrt
import os # file and path operrations
import time
import sys # get input arguments  of the python  program

import plotly
import plotly.graph_objs as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
from plotly.offline.offline import _plot_html

import glob #listing of files
import json

data_path = r"../out/*/"
output_path = r"../temp/"
png_resolution_dpi = 300

directories = glob.glob(data_path)
ic_cycle_order = []

sorted_directories = sorted(directories)#sorted( directories, key = lambda directory: os.path.getctime(directory)) #getmtime for modified time
print(sorted_directories)
# # for directory in sorted_directories:
 # # print(directory+'\n')#print("{} - {}".format(directory, time.ctime(os.path.getctime(directory))) )

channel = int(sys.argv[1])
print('channel = \n', channel);
ic_cycle = int(sys.argv[2])
print('ic_cycle = \n', ic_cycle);
wb_cycle = int(sys.argv[3])
print('wb_cycle = \n', wb_cycle);
wb_sub_cycle = int(sys.argv[4])
print('wb_sub_cycle = \n', wb_sub_cycle);

total_ic_cycles = len(directories) # may  add additional checks  to avoid parasite folders added for testing
# may also check json file to get metadata
# manage the  cases  where user inputs  exceed the limits ic and wb
# total_wb_cycles = #get this from json file

if ic_cycle > (total_ic_cycles - 1) :
	ic_cycle = total_ic_cycles - 1 # this is to  display the  last data

# confirm the data exist

stream_filename = sorted_directories[ic_cycle] + "ch_" + str(channel) + "_raw.dat"
print(stream_filename);

result = sorted_directories[ic_cycle].find('IC_CYCLE')
if result == -1:
	print('Folder name incorrect \n')
	sys.exit()

with open(sorted_directories[ic_cycle]+'cfg.json') as json_file:
	data = json.load(json_file)
	sampling_period_ns = data['Oscilloscopes']['Picoscope 4444']['Sampling Period NS']
	resolution_bits = data['Oscilloscopes']['Picoscope 4444']['Sample Resolution bits']
	voltage_range_str = data['Oscilloscopes']['Picoscope 4444']['Channels']['Channel '+str(channel)]['Voltage Range']
	total_samples_per_waveform = data['Oscilloscopes']['Picoscope 4444']['Channels']['Channel '+str(channel)]['Waveform Number of Samples']
	waveforms_per_wb_cycle = data['Oscilloscopes']['Picoscope 4444']['Channels']['Channel '+str(channel)]['Waveforms per WB Cycle']

	range_val, range_unit = voltage_range_str.split()
	range_val_mv = 0
	if range_unit == 'V':
		range_val_mv = 1000 * int(range_val)
	else:
		range_val_mv = int(range_val)
	print ('range_mv =', range_val_mv)

with open(sorted_directories[ic_cycle]+'stat.json') as json_file:
	data = json.load(json_file)
	captured_waveforms = data['Oscilloscopes']['Picoscope 4444']['Channels']['Channel '+str(channel)]['Waveforms found']

# get the following data from json
wb_cycle_total_samples = waveforms_per_wb_cycle * total_samples_per_waveform
i = 0
sample_offset = wb_cycle * wb_cycle_total_samples + wb_sub_cycle * total_samples_per_waveform

exists = os.path.isfile(stream_filename)
if exists:
	# Store configuration file values
	test = 0
else:
	# Keep presets
	print('ch1 file doesn t exist \n')
	sys.exit()

fp_stream = open(stream_filename, "rb")
fp_stream.seek(sample_offset * 2)
stream1 = np.fromfile(fp_stream, dtype=([('channel', '<i2')]), count = total_samples_per_waveform)
fp_stream.close()

#adc_to_mv( return (raw * inputRanges[rangeIndex]) / unit->maxADCValue;
data = np.float64(stream1['channel'])
data = data * range_val_mv / 32768

# # Create a trace
trace = go.Scatter(
    y = stream1['channel']
)
layout = go.Layout(
    margin=dict(
        l=0,
        r=0,
        b=0,
        t=0,
		pad=4
    ),
    paper_bgcolor='#7f7f7f',
    plot_bgcolor='#c7c7c7'
)
fig = plt.figure()
fig = go.Figure(data=[trace], layout=layout)
plotly.offline.plot(fig, filename= output_path + 'ch_' + str(channel) + ".html", auto_open=False)


import logging
import serial
import time

###################################################################################

class CommUART(object):
    def __init__(self, address):
        self.address = address
        self.sc = None

    def connect(self):
        logging.debug("Opening COM port : {0}".format(self.address))
        self.sc = None
        while self.sc is None:
            try:
                self.sc = serial.Serial(port=self.address, baudrate=3000000, rtscts=False)
            except serial.serialutil.SerialException as se:
                if 'Device or resource busy:' in se.__str__():
                    logging.info('Opening COM port is taking a little while, please stand by...')
                else:
                    logging.error('se: {0}'.format(se))
                time.sleep(1)
        logging.debug("COM port open successfully.")
        self.sc.flushInput()

    def disconnect(self):
        logging.debug("Closing COM port : {0}".format(self.address))
        self.sc.close()

    def receivedPacket(self, length):
        if self.sc is None:
            raise Exception('COM port is not opened.')
        packet = b''
        received = 0
        while received < length:
            serialByte = self.sc.read(1)
            if serialByte is None:
                raise Exception('Bad character.')
            elif len(serialByte) == 0:
                break
            elif received < length:
                received += 1
                packet += serialByte
        return packet

    def send(self, data):
        self.sc.write(bytes([data]))

    def prbs8(self, curval):
        newbit = (((curval >> 6) ^ (curval >> 5)) & 1)
        return ((curval << 1) | newbit) & 0x7f
###################################################################################

def main():
    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s : %(message)s')

    #comm = CommUART("/dev/cu.usbserial-FT0NCE8B")
    #comm = CommUART("/dev/cu.usbmodem143422")
    comm = CommUART("/dev/cu.usbmodem143132")
    comm.connect()

    curval = 0
    #packet = comm.receivedPacket(1)
    #curval = int.from_bytes(packet, byteorder = 'little')
    val = comm.prbs8(0xff)
    byteCount = 0
    dropcnt = 0
    deltatime = 0
    drop = False
    while True:
        try:
            comm.send(val)
            startTime = time.time()
#            packet = comm1.receivedPacket(1)
            endTime = time.time()
            deltatime += endTime - startTime
 #           curval = int.from_bytes(packet, byteorder = 'little')
 #           if curval != val:
 #               dropcnt += 1
            val = comm.prbs8(val)
            byteCount += 1

            if deltatime > 0:
                bytesPerSec = byteCount / deltatime #(endTime - startTime)

            #print("Bytes : {0}".format(bytes))
            #if drop:
            #    print("Dropped.... Bytes/sec : {0}".format(bytesPerSec))
            #else:
            if (byteCount & 0xff) == 0:
                print("Bytes/sec : %.2f, drop %d " %(bytesPerSec, dropcnt))
        except KeyboardInterrupt:
            print("KeyboardInterrupt. Exiting.")
            break

    comm.disconnect()
 #   comm1.disconnect()
###################################################################################

if __name__ == '__main__':
    main()


import matplotlib.pyplot as plt
import numpy as np
import matplotlib.ticker as ticker

# result_file = "../expt/results/task4_1.txt"
result_file = "finalT2.txt"#"outputDataT2.txt"
instance_list = ["../instances/i-1.txt","../instances/i-2.txt","../instances/i-3.txt"]
# instance_list = ["i-1.txt","i-2.txt","i-3.txt"]
algorithms = ["thompson-sampling", "thompson-sampling"]
final_dict = {"../instances/i-1.txt":{}, "../instances/i-2.txt":{}, "../instances/i-3.txt":{}}
horizons =  [100, 400, 1600, 6400, 25600, 102400]
# final_dict = {"i-3.txt":{}, "i-1.txt":{}, "i-2.txt":{}}
with open(result_file,'r') as f:
    lines = f.readlines()
    for line in lines:
        x = line.rstrip().split(', ')
        if x[1] in final_dict[x[0]].keys():
            final_dict[x[0]][x[1]][int(np.log2(int(x[4])/100)/2)] += float(x[5])/50.0
        else:
            final_dict[x[0]][x[1]] = [0]*6
            final_dict[x[0]][x[1]][int(np.log2(int(x[4])/100)/2)] += float(x[5])/50.0

for i,instance in enumerate(instance_list):
    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.set_xscale('log')
    ax.xaxis.set_ticks(horizons)
    ax.get_xaxis().set_major_formatter(ticker.ScalarFormatter())
    ts = final_dict[instance]["thompson-sampling"]
    plt.plot(horizons,ts,label="Thompson-Sampling")
    ts = final_dict[instance]["thompson-sampling-with-hint"]
    plt.plot(horizons,ts,label="Thompson-Sampling (with Hint)")
    plt.xlabel("Horizon (Logarithmic Scale, Base 2)")
    plt.ylabel("Average Regret")
    plt.legend()

    pltTitle = instance.replace('../instances/', '')
    pltTitle = pltTitle.replace('.txt', '')
    pltTitle = pltTitle.replace('i-', 'Instance ')
    # plt.plot(x_axis,kl_ucb,x_axis,ts,x_axis,ucb,x_axis,eg)
    plt.title("{}".format(pltTitle))
    plt.savefig("testT2_instance{}.png".format(i+1))


# Example solution for HW 4

# %%
# Import the modules we will use
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# %%
# ** MODIFY **
# Set the file name and path to where you have stored the data
filename = 'streamflow_week5.txt'
filepath = os.path.join('data', filename)
print(os.getcwd())
print(filepath)

# filepath = '../Assignments/Solutions/data/streamflow_week1.txt'

# %%
#Read the data into a pandas dataframe
data=pd.read_table(filepath, sep = '\t', skiprows=30,
        names=['agency_cd', 'site_no', 'datetime', 'flow', 'code']
        )

# Expand the dates to year month day
data[["year", "month", "day"]] =data["datetime"].str.split("-", expand=True)
data['year'] = data['year'].astype(int)
data['month'] = data['month'].astype(int)
data['day'] = data['day'].astype(int)

# %%
# Sorry no more helpers past here this week, you are on your own now :)
# Hints - you will need the functions: describe, info, groupby, sort, head and tail.
#%%
# Esimation5
data.flow
# print(data.datetime[(data.flow >= rnge[0]) & (data.flow <= rnge[1])])

# list_2010 = []
# list_2011 = []
# list_2012 = []
# list_2013 = []
# list_2014 = []
# list_2015 = []
# list_2016 = []
# list_2017 = []
# list_2016 = []
# list_2017 = []
# list_2020 = []

# data.flow[(data.year >= 2010) & (data.year <= 2020) & (data.month == i)].mean]
#%%
# 1, 3, 5, 7, 8, 10, 12
for d in range(2010, 2020):
        fig1 = plt.figure()
        fig1.patch.set_facecolor('xkcd:mint green')
        plt.title('%d'%(d))
        plt.ylabel('flow')
        for i in (1, 3, 5, 7, 8, 10, 12):
                # print(i)
                # print(data.flow[(data.year == 2010) & (data.month == i)].mean)
                # print("\n")
                # data.flow[(data.year == 2010) & (data.month == i)]
                x = list(range(1, 32))
                plt.plot(x, (data.flow[(data.year == d) & (data.month == i)]))
                plt.xlabel('days in month')
                plt.legend(['1', '3', '5', '7', '8', '10', '12'])
                plt.savefig('graphs/flow-set1_%d'%(d))

# x = list(range(1, 32))
# print(x)

# print(flow_data.size)
# print(flow_data.shape)
# flow_202009 = flow_data[11571:11585, 3]
# print(flow_202009)

# x = [6.,7,8,9,10,11,12,13,14,15,16,17,18,19]
# fig9 = plt.figure()
# fig9.patch.set_facecolor('xkcd:mint green')
# plt.plot(x, flow_202009)
# plt.xlabel('days in September 2020')
# plt.ylabel('flow')
# plt.legend()
# plt.savefig('graphs/flow_202009')

# %%
# 4, 6, 9, 11
for d in range(2010, 2020):
        fig2 = plt.figure()
        fig2.patch.set_facecolor('xkcd:mint green')
        plt.title('%d'%(d))
        plt.ylabel('flow')
        for i in (4, 6, 9, 11):
                # print(i)
                # print(data.flow[(data.year == 2010) & (data.month == i)].mean)
                # print("\n")
                # data.flow[(data.year == 2010) & (data.month == i)]
                x = list(range(1, 31))
                # print(x)
                plt.plot(x, (data.flow[(data.year == d) & (data.month == i)]))
                plt.xlabel('days in the month')
                plt.legend(['4', '6', '9', '11'])
                plt.savefig('graphs/flow-set2_%d'%(d))
# %%
# 2020

fig3 = plt.figure()
fig3.patch.set_facecolor('xkcd:mint green')
plt.title('2020')
plt.ylabel('flow')
for i in (1, 3, 5, 7, 8):
        # print(i)
        # print(data.flow[(data.year == 2010) & (data.month == i)].mean)
        # print("\n")
        # data.flow[(data.year == 2010) & (data.month == i)]
        x = list(range(1, 32))
        # print(x)
        plt.plot(x, (data.flow[(data.year == 2020) & (data.month == i)]))
        plt.xlabel('days in the month')
        plt.legend(['1', '3', '5', '7', '8'])
        plt.savefig('graphs/flow-set3_2020-%i'%(i))

data.flow[(data.year == 2020) & (data.month == 1)]

fig4 = plt.figure()
fig4.patch.set_facecolor('xkcd:mint green')
plt.title('2020')
plt.ylabel('flow')
for i in (4, 6):
        # print(i)
        # print(data.flow[(data.year == 2010) & (data.month == i)].mean)
        # print("\n")
        # data.flow[(data.year == 2010) & (data.month == i)]
        x = list(range(1, 31))
        # print(x)
        plt.plot(x, (data.flow[(data.year == 2020) & (data.month == i)]))
        plt.xlabel('days in the month')
        plt.legend(['4', '6'])
        plt.savefig('graphs/flow-set4_2020-%i'%(i))
# %%
# When September Ends
x = list(range(1, 27))
fig5 = plt.figure()
fig5.patch.set_facecolor('xkcd:mint green')
plt.title('2020-9')
plt.ylabel('flow')
plt.plot(x, (data.flow[(data.year == 2020) & (data.month == 9)]))
plt.xlabel('days in the month')
plt.legend([])
plt.savefig('graphs/flow-set5_2020-9')

print((data.flow[(data.year == 2020) & (data.month == 9)]))
# %%


import os
import sys
import torch
from collections import defaultdict
import json
import pickle
import logging
logging.basicConfig()

sys.path.append(os.getcwd())
from generalization_config import config
from utils.graph_utils import create_adj_matrix, blockify_A, create_coordinate_channel, \
    create_edge_index_from_adjacency_matrix
from utils.training_gcn_utils import validate
from utils.training_unet_utils import validate as validate_unet
from utils.videoloader import trafic4cast_dataset

from models.unet import UNet
from models.graph_models import KipfNet_orig, KipfNet, KipfNetd2, Graph_resnet

def get_graphdata_obj(inputs, edge_index, y, num_features=38, num_classes=9):
    graphdata = Data(x=inputs, edge_index=edge_index, y=y)

    return graphdata

def get_n_params(model):
    pp = 0
    for p in list(model.parameters()):
        nn = 1
        for s in list(p.size()):
            nn = nn * s
        pp += nn
    return pp

if __name__ == "__main__":
    print('batch_size: ', config['dataloader']['batch_size'])
    print(config['device_num'])
    device = torch.device(config['device_num'])

    model_tuple_list = config['model_tuple_list']

    resultdict = defaultdict(dict)

    for city in ['Berlin', 'Moscow', 'Istanbul']:
        config['dataset']['cities'] = [city]

        dataset_val = trafic4cast_dataset(split_type='validation', **config['dataset'],
                                          reduce=True, filter_test_times=True)

        val_loader = torch.utils.data.DataLoader(dataset_val, shuffle=False,
                                                 **config['dataloader'])

        for model_tuple in model_tuple_list:
            model_plot_name = model_tuple[0]
            model_path = model_tuple[1]
            is_graph = model_tuple[2]
            graph_model_name = model_tuple[3]

            with open(os.path.join(model_path, 'config.json'), 'r') as f:
                model_config = json.load(f)

                adj, nn_ixs, G, mask = create_adj_matrix(city=config['dataset']['cities'][0],
                                                         mask_threshold=config['mask_threshold'])

                if not is_graph:
                    model_config['model']['batch_norm'] = True
                    model = UNet(**model_config['model']).to(device)
                    model.load_state_dict(torch.load(os.path.join(model_path, 'checkpoint.pt'),
                                                     map_location=device))

                    mask = torch.from_numpy(mask).to(device)

                    if 'MIE-Lab' in model_plot_name:
                        norm = False
                    else:
                        norm = True

                    val_loss = validate_unet(model=model, val_loader=val_loader, device=device, mask=mask,
                                             config=model_config, print_loss=False, norm=norm)

                if is_graph:

                    n_features = 38
                    batch_size = config['dataloader']['batch_size']
                    assert batch_size == 1, "batch_size should be 1 for graphs"

                    coords = create_coordinate_channel(b=batch_size)

                    if config['dataloader']['batch_size'] > 1:
                        adj = blockify_A(adj, config['dataloader']['batch_size'])

                    edge_index = create_edge_index_from_adjacency_matrix(adj)
                    edge_index = edge_index.to(device)

                    if graph_model_name == 'kipfnet':
                        model = KipfNet_orig(num_features=n_features,
                                             num_classes=9, **model_config['model']['KIPF']).to(device)
                        model.load_state_dict(torch.load(os.path.join(model_path, 'checkpoint.pt'),
                                                         map_location=device))

                    elif graph_model_name == 'skipfnet':
                        model = KipfNet(num_features=n_features,
                                        num_classes=9, **model_config['model']['KipfNet']).to(device)
                        model.load_state_dict(torch.load(os.path.join(model_path, 'checkpoint.pt'),
                                                         map_location=device))

                    elif graph_model_name == 'skipfnet2d':
                        model = KipfNetd2(num_features=n_features,
                                          num_classes=9, **model_config['model']['KipfNetd2']).to(device)
                        model.load_state_dict(torch.load(os.path.join(model_path, 'checkpoint.pt'),
                                                         map_location=device))

                    elif graph_model_name == 'Graph_resnet':
                        model = Graph_resnet(num_features=n_features,
                                             num_classes=9, **model_config['model']['Graph_resnet']).to(device)
                        model.load_state_dict(torch.load(os.path.join(model_path, 'checkpoint.pt'),
                                                         map_location=device))

                    mask = None
                    val_loss = validate(model=model, val_loader=val_loader, device=device,
                                        adj=adj, nn_ixs=nn_ixs, edge_index=edge_index, coords=coords,
                                        mask=mask, batch_size=batch_size, print_loss=False)

                print("Validation loss {}: {} = {:.2f}".format(city, model_plot_name, val_loss))
                resultdict[model_plot_name][city] = val_loss

                nb_params = get_n_params(model)
                resultdict[model_plot_name]['nb_params'] = nb_params

    pickle.dump(resultdict, open(os.path.join('.', 'output', 'data_generalization.p'), 'wb'))


# Copyright 2015 IBM Corp.
#
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import unittest

import mock

import pypowervm.entities as ent
import pypowervm.tasks.cluster_ssp as cs
import pypowervm.tests.tasks.util as tju
import pypowervm.util as u
import pypowervm.wrappers.cluster as clust
import pypowervm.wrappers.job as jwrap
import pypowervm.wrappers.storage as stor

CREATE_CLUSTER = 'cluster_create_job_template.txt'

class TestClusterSSP(unittest.TestCase):

    @mock.patch('pypowervm.wrappers.job.Job.delete_job')
    @mock.patch('pypowervm.wrappers.job.Job._monitor_job')
    @mock.patch('pypowervm.wrappers.job.Job.job_status')
    @mock.patch('pypowervm.adapter.Adapter')
    def test_crt_cluster_ssp(self, mock_adp, mock_status, mock_monitor_job,
                             mock_del_job):
        # Load up GET Cluster/do/Create (job template)
        mock_adp.read.return_value = tju.load_file(CREATE_CLUSTER, mock_adp)
        # We'll pretend the job ran and completed successfully
        mock_monitor_job.return_value = False
        mock_status.__get__ = mock.Mock(
            return_value=jwrap.JobStatus.COMPLETED_OK)

        # Mock Job.create_job to check job parameter values
        def create_job(job_el, entry_type, *args, **kwargs):
            self.assertEqual(entry_type, clust.Cluster.schema_type)
            job = jwrap.Job.wrap(ent.Entry({}, job_el, None))
            param_vals = job._get_vals(u.xpath(
                'JobParameters', 'JobParameter', 'ParameterValue'))
            self.assertEqual(
                param_vals[0],
                '<uom:Cluster xmlns:uom="http://www.ibm.com/xmlns/systems/powe'
                'r/firmware/uom/mc/2012_10/" schemaVersion="V1_0"><uom:Metadat'
                'a><uom:Atom/></uom:Metadata><uom:ClusterName>clust_name</uom:'
                'ClusterName><uom:RepositoryDisk schemaVersion="V1_0"><uom:Phy'
                'sicalVolume schemaVersion="V1_0"><uom:Metadata><uom:Atom/></u'
                'om:Metadata><uom:VolumeName>repos_pv_name</uom:VolumeName></u'
                'om:PhysicalVolume></uom:RepositoryDisk><uom:Node schemaVersio'
                'n="V1_0"><uom:Node schemaVersion="V1_0"><uom:Metadata><uom:At'
                'om/></uom:Metadata><uom:HostName>vios1</uom:HostName><uom:Par'
                'titionID>5</uom:PartitionID><uom:MachineTypeModelAndSerialNum'
                'ber schemaVersion="V1_0"><uom:Metadata><uom:Atom/></uom:Metad'
                'ata><uom:MachineType>XXXX</uom:MachineType><uom:Model>YYY</uo'
                'm:Model><uom:SerialNumber>ZZZZZZZ</uom:SerialNumber></uom:Mac'
                'hineTypeModelAndSerialNumber><uom:VirtualIOServer href="https'
                '://a.example.com:12443/rest/api/uom/VirtualIOServer/12345678-'
                '1234-1234-1234-123456789012" rel="related"/></uom:Node></uom:'
                'Node></uom:Cluster>')
            self.assertEqual(
                param_vals[1],
                '<uom:SharedStoragePool xmlns:uom="http://www.ibm.com/xmlns/sy'
                'stems/power/firmware/uom/mc/2012_10/" schemaVersion="V1_0"><u'
                'om:Metadata><uom:Atom/></uom:Metadata><uom:PhysicalVolumes sc'
                'hemaVersion="V1_0"><uom:PhysicalVolume schemaVersion="V1_0"><'
                'uom:Metadata><uom:Atom/></uom:Metadata><uom:VolumeName>hdisk1'
                '</uom:VolumeName></uom:PhysicalVolume><uom:PhysicalVolume sch'
                'emaVersion="V1_0"><uom:Metadata><uom:Atom/></uom:Metadata><uo'
                'm:VolumeName>hdisk2</uom:VolumeName></uom:PhysicalVolume><uom'
                ':PhysicalVolume schemaVersion="V1_0"><uom:Metadata><uom:Atom/'
                '></uom:Metadata><uom:VolumeName>hdisk3</uom:VolumeName></uom:'
                'PhysicalVolume></uom:PhysicalVolumes><uom:StoragePoolName>ssp'
                '_name</uom:StoragePoolName></uom:SharedStoragePool>')
            return mock.MagicMock()
        mock_adp.create_job.side_effect = create_job
        node = clust.Node.bld(
            mock_adp, hostname='vios1', lpar_id=5, mtms='XXXX-YYY*ZZZZZZZ',
            vios_uri='https://a.example.com:12443/rest/api/uom/VirtualIOServe'
            'r/12345678-1234-1234-1234-123456789012')
        repos = stor.PV.bld(mock_adp, name='repos_pv_name')
        data = [stor.PV.bld(mock_adp, name=n) for n in (
            'hdisk1', 'hdisk2', 'hdisk3')]
        cs.crt_cluster_ssp('clust_name', 'ssp_name', repos, node, data)
        # run_job() should run delete_job() at the end
        self.assertEqual(mock_del_job.call_count, 1)


from copy import deepcopy
from opengever.base.schema import TableChoice
from opengever.oneoffixx import _
from opengever.oneoffixx.api_client import OneoffixxAPIClient
from opengever.oneoffixx.command import CreateDocumentFromOneOffixxTemplateCommand
from opengever.oneoffixx.utils import whitelisted_template_types
from plone.i18n.normalizer.interfaces import IFileNameNormalizer
from plone.supermodel import model
from plone.z3cform.layout import FormWrapper
from z3c.form import button
from z3c.form.field import Fields
from z3c.form.form import Form
from zope import schema
from zope.component import getUtility
from zope.i18n import translate
from zope.interface import provider
from zope.schema.interfaces import IContextSourceBinder
from zope.schema.vocabulary import SimpleVocabulary

def get_oneoffixx_favorites():
    """Return the user chosen favorites as a template group, if any."""
    api_client = OneoffixxAPIClient()
    favorites = api_client.get_oneoffixx_favorites()
    if favorites.get('templates'):
        return favorites
    return None

def get_oneoffixx_template_groups():
    """Return the template groups.

    Potentially amended with user chosen favorites.
    """
    api_client = OneoffixxAPIClient()
    # We need to work on a copy to not pollute the cached one
    template_groups = deepcopy(api_client.get_oneoffixx_template_groups())
    favorites = get_oneoffixx_favorites()
    if favorites:
        template_groups.insert(0, favorites)
    return template_groups

def get_oneoffixx_templates():
    """Return all oneoffixx templates.

    We do not want duplicates from favorites here.
    """
    api_client = OneoffixxAPIClient()
    return (
        OneOffixxTemplate(template, template_group.get('localizedName', ''))
        for template_group in api_client.get_oneoffixx_template_groups()
        for template in template_group.get("templates")
        if template.get('metaTemplateId') in whitelisted_template_types
    )

def default_template_group():
    """Return all templates, or the user favorites, if defined by user."""
    favorites = get_oneoffixx_favorites()
    if favorites:
        return favorites.get('id')
    return None

@provider(IContextSourceBinder)
def list_templates(context):
    """Return a list available templates."""
    templates = get_oneoffixx_templates()
    template_group = context.REQUEST.form.get('form.widgets.template_group')
    terms = []

    for template in templates:
        terms.append(SimpleVocabulary.createTerm(
            template, template.template_id, template.title))

    # We filter templates when template_group has been selected
    if template_group is not None:
        favorites = get_oneoffixx_favorites()
        # Favorites are a special case
        if favorites and template_group[0] == favorites.get('id'):
            terms = [
                SimpleVocabulary.createTerm(
                    OneOffixxTemplate(
                        template, favorites.get('localizedName', '')),
                    template.get('id'),
                    template.get('localizedName'),
                )
                for template in favorites.get('templates')
            ]
        elif template_group[0] != '--NOVALUE--':
            terms = [term for term in terms if term.value.group == template_group[0]]

    return MutableObjectVocabulary(terms)

@provider(IContextSourceBinder)
def list_template_groups(context):
    """Return the list of available template groups."""
    template_groups = get_oneoffixx_template_groups()
    terms = []
    for group in template_groups:
        terms.append(SimpleVocabulary.createTerm(group.get("id"),
                                                 group.get("id"),
                                                 group.get("localizedName")))
    return MutableObjectVocabulary(terms)

class OneOffixxTemplate(object):

    def __init__(self, template, groupname):
        self.title = template.get("localizedName")
        self.template_id = template.get("id")
        self.group = template.get('templateGroupId')
        self.groupname = groupname
        template_type = template['metaTemplateId']
        template_type_info = whitelisted_template_types[template_type]
        self.content_type = template_type_info['content-type']
        filename = template.get("localizedName")
        normalizer = getUtility(IFileNameNormalizer, name='gever_filename_normalizer')
        self.filename = normalizer.normalize(filename, extension=template_type_info['extension'])
        self.languages = template.get("languages")

    def __eq__(self, other):
        if type(other) == type(self):
            return self.template_id == other.template_id
        return False

class MutableObjectVocabulary(SimpleVocabulary):

    def __contains__(self, value):
        try:
            return any([value == val for val in self.by_value])
        except TypeError:
            return False

class ICreateDocumentFromOneOffixxTemplate(model.Schema):

    # XXX - this always renders the --NOVALUE-- as the actually chosen
    # default is actually loaded over AJAX - confusing and bad UX
    template_group = schema.Choice(
        title=_(u'label_template_group', default=u'Template group'),
        source=list_template_groups,
        required=False,
        defaultFactory=default_template_group,
    )

    template = TableChoice(
        title=_(u"label_template", default=u"Template"),
        source=list_templates,
        required=True,
        show_filter=True,
        vocabulary_depends_on=['form.widgets.template_group'],
        columns=(
            {'column': 'title',
             'column_title': _(u'label_title', default=u'Title'),
             'sort_index': 'sortable_title'},
            )
    )

    title = schema.TextLine(
        title=_(u"label_title", default=u"Title"),
        required=True)

class SelectOneOffixxTemplateDocumentWizardStep(Form):

    label = _(u'create_document_with_template', default=u'Create document from template')
    ignoreContext = True
    fields = Fields(ICreateDocumentFromOneOffixxTemplate)

    def updateWidgets(self, prefix=None):
        super(SelectOneOffixxTemplateDocumentWizardStep, self).updateWidgets(prefix=prefix)
        self.widgets['template_group'].noValueMessage = translate(
            _(u'label_all_template_groups', default=u'All templates'), context=self.request)

    def finish_document_creation(self, data):
        new_doc = self.create_document(data)
        self.activate_external_editing(new_doc)
        return self.request.RESPONSE.redirect(new_doc.absolute_url())

    def activate_external_editing(self, new_doc):
        """Add the oneoffixx external_editor URL to redirector queue."""
        new_doc.setup_external_edit_redirect(self.request, action="oneoffixx")

    def create_document(self, data):
        """Create a new document based on a template."""
        command = CreateDocumentFromOneOffixxTemplateCommand(self.context, data['title'], data['template'])
        return command.execute()

    @button.buttonAndHandler(_('button_save', default=u'Save'), name='save')
    def handleApply(self, action):
        data, errors = self.extractData()

        if not errors:
            return self.finish_document_creation(data)

        self.status = self.formErrorsMessage
        return None

    @button.buttonAndHandler(_(u'button_cancel', default=u'Cancel'), name='cancel')
    def cancel(self, action):
        return self.request.RESPONSE.redirect(self.context.absolute_url())

class SelectOneOffixxTemplateDocumentView(FormWrapper):

    form = SelectOneOffixxTemplateDocumentWizardStep


import pdb
import sys
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

sys.path.append("..")
from utils.misc import ixvr

class AttendFeedForward(nn.Module):
	"""
	Similiar to the attend (Section 3.1) module of the DecAtt paper
	"""
	def __init__(self, inp_size, hidden_size=200):
		super(AttendFeedForward, self).__init__()

		self.hidden_size = hidden_size
		self.linear = nn.Sequential( \
			nn.Linear(inp_size, hidden_size), \
			nn.ReLU(), \
			nn.BatchNorm1d(num_features=hidden_size), \
			nn.Linear(hidden_size, hidden_size), \
			nn.ReLU(), \
			nn.BatchNorm1d(num_features=hidden_size))

	def forward(self, s1, s2, mask1, mask2):
		"""
		Args:
			s1: Sentence 1 BiLSTM embeddings (b x LA x inp_size)
			s2: Sentence 2 BiLSTM embeddings (b x LB x inp_size)
			mask1: Sentence 1 mask (b x LA)
			mask2: Sentence 2 mask (b x LB)
		Output:
			alphas: Soft aligned combinations of s1 w.r.t. s2 tokens (b x maxlen x inp_size)
			betas: Soft aligned combinations of s2 w.r.t. s1 tokens (b x maxlen x inp_size)
		"""
		batch_size = s1.shape[0]
		maxlen = s1.shape[1]
		inp_size = s1.shape[2]

		h1 = self.linear(s1.view(-1, inp_size)).view(batch_size, maxlen, -1)
		# b x LA x hidden_size
		h2 = self.linear(s2.view(-1, inp_size)).view(batch_size, maxlen, -1)
		# b x LB x hidden_size
		h2t = torch.transpose(h2, 1, 2)
		# b x hidden_size x LB

		e = torch.bmm(h1, h2t)

		e_alpha = torch.mul(e, mask1.unsqueeze(-1))
		e_alpha = torch.exp(e_alpha - torch.max(e_alpha, dim=1)[0].unsqueeze(1))
		e_alpha = torch.div(e_alpha, torch.sum(e_alpha, dim=1).unsqueeze(1))
		# b x LA x LB

		e_beta = torch.mul(e, mask2.unsqueeze(1))
		e_beta = torch.exp(e_beta - torch.max(e_beta, dim=2)[0].unsqueeze(-1))
		e_beta = torch.div(e_beta, torch.sum(e_beta, dim=2).unsqueeze(-1))
		# b x LA x LB

		alphas = torch.bmm(torch.transpose(e_alpha, 1, 2), s1)
		alphas = torch.mul(alphas, mask2.unsqueeze(-1))
		# b x LB x inp_size
		betas = torch.bmm(e_beta, s2)
		betas = torch.mul(betas, mask1.unsqueeze(-1))
		# b x LA x inp_size

		return alphas, betas

class CompareFeedForward(nn.Module):
	"""
	Similar to the compare (Section 3.2) module of the DecAtt paper
	except instead of returning the sum of the embeddings v1 and v2
	(which might be susceptible to the length of the sequence),
	this returns v1_avg, v1_max, v2_avg, v2_max.
	"""
	def __init__(self, inp_size, hidden_size=200):
		super(CompareFeedForward, self).__init__()

		self.linear = nn.Sequential( \
			nn.Linear(inp_size * 2, hidden_size), \
			nn.ReLU(), \
			nn.BatchNorm1d(num_features=hidden_size), \
			nn.Linear(hidden_size, hidden_size), \
			nn.ReLU(), \
			nn.BatchNorm1d(num_features=hidden_size))

	def forward(self, s1, s2, alphas, betas, mask1, mask2):
		"""
		Args:
			s1: Sentence 1 BiLSTM embeddings (b x LA x inp_size)
			s2: Sentence 2 BiLSTM embeddings (b x LB x inp_size)
			alphas: Aligned phrases (b x LB x inp_size)
			betas: Aligned phrases (b x LA x inp_size)
			mask1: Sentence 1 mask (b x LA)
			mask2: Sentence 2 mask (b x LB)
		Output:
			v1_avg: Comparison avg. pooled vector for aligned sentence s1 (b x hidden_size)
			v1_max: Comparison max. pooled vector for aligned sentence s1 (b x hidden_size)
			v2_avg: Comparison avg. pooled vector for aligned sentence s2 (b x hidden_size)
			v2_max: Comparison max. pooled vector for aligned sentence s2 (b x hidden_size)
		"""
		batch_size = s1.shape[0]
		maxlen = s1.shape[1]
		inp_size = s1.shape[2]

		in1 = torch.cat((s1, betas), dim=2)
		# b x LA x (inp_size * 2)
		in2 = torch.cat((s2, alphas), dim=2)
		# b x LB x (inp_size * 2)

		v1 = self.linear(in1.view(-1, inp_size * 2)).view(batch_size, maxlen, -1)
		# b x LA x hidden_size
		v1_avg = torch.sum(torch.mul(v1, mask1.unsqueeze(-1)), dim=1)
		v1_avg = torch.div(v1_avg, torch.sum(mask1, dim=1).unsqueeze(-1))
		# b x hidden_size
		v1_max = torch.max(torch.mul(v1, mask1.unsqueeze(-1)), dim=1)[0]
		# b x hidden_size

		v2 = self.linear(in2.view(-1, inp_size * 2)).view(batch_size, maxlen, -1)
		# b x LB x hidden_size
		v2_avg = torch.sum(torch.mul(v2, mask2.unsqueeze(-1)), dim=1)
		v2_avg = torch.div(v2_avg, torch.sum(mask2, dim=1).unsqueeze(-1))
		# b x hidden_size
		v2_max = torch.max(torch.mul(v2, mask2.unsqueeze(-1)), dim=1)[0]
		# b x hidden_size

		return v1_avg, v1_max, v2_avg, v2_max

class ESIMBNMultiTask(nn.Module):
	"""
	Model architecture similar to the Enhanced Sequential Inference Model (ESIM)
	as described in https://arxiv.org/abs/1609.06038 without the Tree LSTM. This
	model also has BatchNorm layers for preventing overfitting instead of dropout
	layers.

	The BatchNorm order followed here is LIN -> ReLU -> BN even though the original
	paper used BN before the non-linearity. Some online sources claim that BN after
	the ReLU gives better results.

	The model is designed for both Reddit response prediction task and Quora
	semantic question matching task.
	"""
	def __init__(self, hidden_size=200, glove_loader=None, pretrained_emb=True):
		"""
		Args:
			hidden_size: Size of the intermediate linear layers
			glove_loader: GLoVe embedding loader
			pretrained_emb: Use pretrained embeddings
		"""
		super(ESIMBNMultiTask, self).__init__()

		if not pretrained_emb:
			raise NotImplementedError('always loads pretrained embeddings')

		word_vectors = glove_loader.word_vectors
		word_vectors = np.vstack(word_vectors)
		vocab_size = word_vectors.shape[0]
		embed_size = word_vectors.shape[1]

		self.embedding = nn.Embedding(vocab_size, embed_size)
		self.embedding.load_state_dict({'weight': torch.Tensor(word_vectors)})
		self.encoder = nn.LSTM(input_size=embed_size, hidden_size=hidden_size, num_layers=1, bidirectional=True)
		self.attend = AttendFeedForward(inp_size=hidden_size * 2, hidden_size=hidden_size)
		self.compare = CompareFeedForward(inp_size=hidden_size * 2, hidden_size=hidden_size)

		# prediction layer for the Quora task
		self.sts_pred = nn.Sequential( \
			nn.Linear(hidden_size * 4, hidden_size), \
			nn.ReLU(), \
			nn.BatchNorm1d(num_features=hidden_size), \
			nn.Linear(hidden_size, 2))

		# tranformation layer for the response
		self.response_transform = nn.Sequential( \
			nn.Linear(hidden_size * 2, hidden_size * 2), \
			nn.ReLU(), \
			nn.BatchNorm1d(num_features=hidden_size * 2), \
			nn.Linear(hidden_size * 2, hidden_size * 2), \
			nn.BatchNorm1d(num_features=hidden_size * 2))

		self.reset_parameters()

	def reset_parameters(self):
		"""Initialize network weights using Xavier init (with bias 0.01)"""
		self.apply(ixvr)

	def forward(self, s1, s2, len1, len2):
		"""
		Args:
			s1: Sentence 1 embeddings (b x LA)
			s2: Sentence 2 embeddings (b x LB)
			len1: Sentence 1 length (b)
			len2: Sentence 2 length (b)
		"""
		batch_size = s1.shape[0]
		maxlen = s1.shape[1]

		s1 = self.embedding(s1).transpose(0, 1)
		s1, _ = self.encoder(s1)
		s1 = torch.transpose(s1, 0, 1).contiguous()
		# b x LA x (hidden_size * 2)
		s2 = self.embedding(s2).transpose(0, 1)
		s2, _ = self.encoder(s2)
		s2 = torch.transpose(s2, 0, 1).contiguous()
		# b x LB x (hidden_size * 2)

		mask1 = torch.arange(0, maxlen).expand(batch_size, maxlen)
		if torch.cuda.is_available():
			mask1 = mask1.cuda()
		mask1 = mask1 < len1.unsqueeze(-1)
		mask2 = torch.arange(0, maxlen).expand(batch_size, maxlen)
		if torch.cuda.is_available():
			mask2 = mask2.cuda()
		mask2 = mask2 < len2.unsqueeze(-1)

		mask1 = mask1.float()
		mask2 = mask2.float()

		alphas, betas = self.attend(s1, s2, mask1, mask2)
		v1_avg, v1_max, v2_avg, v2_max = self.compare(s1, s2, alphas, betas, mask1, mask2)
		assert batch_size > 1
		out = self.sts_pred(torch.cat((v1_avg, v1_max, v2_avg, v2_max), dim=1))

		return out

	def rank_responses(self, q, resp, len_q, len_resp):
		"""
		Args:
			q: Reddit question embeddings (b x LA)
			resp: Reddit response candidates embeddings (b x K x LB)
			len_q: Length of the input question (b)
			len_resp: Length of the response candidates (b x K)
		"""

		batch_size = q.shape[0]
		maxlen = q.shape[1]
		K = resp.shape[1]

		q = self.embedding(q).transpose(0, 1)
		q, _ = self.encoder(q)
		q = torch.transpose(q, 0, 1).contiguous()
		# b x LA x (hidden_size * 2)
		resp = self.embedding(resp).view(batch_size * K, maxlen, -1).transpose(0, 1)
		resp, _ = self.encoder(resp)
		resp = torch.transpose(resp, 0, 1).view(batch_size, K, maxlen, -1).contiguous()
		# b x K x LB x (hidden_size * 2)

		mask1 = torch.arange(0, maxlen).expand(batch_size, maxlen)
		if torch.cuda.is_available():
			mask1 = mask1.cuda()
		mask1 = mask1 < len_q.unsqueeze(-1)
		mask1 = mask1.float()
		# b x LA

		mask2 = torch.arange(0, maxlen).expand(batch_size * K, maxlen)
		if torch.cuda.is_available():
			mask2 = mask2.cuda()
		mask2 = mask2 < len_resp.view(-1).unsqueeze(-1)
		mask2 = mask2.view(batch_size, K, -1).float()
		# b x K x LB

		q = q.unsqueeze(1).expand(-1, K, -1, -1).contiguous().view(batch_size * K, maxlen, -1)
		# (b * K) x LA x (hidden_size * 2)
		mask1 = mask1.unsqueeze(1).expand(-1, K, -1).contiguous().view(batch_size * K, maxlen)
		# (b * K) x LA

		resp = resp.view(batch_size * K, maxlen, -1)
		# (b * K) x LB x (hidden_size * 2)
		mask2 = mask2.view(batch_size * K, maxlen)
		# (b * K) x LB

		alphas, betas = self.attend(q, resp, mask1, mask2)
		v1_avg, v1_max, v2_avg, v2_max = self.compare(q, resp, alphas, betas, mask1, mask2)

		v1 = torch.cat((v1_avg, v1_max), dim=1)
		# (b * K) x (hidden_size * 2)
		v2 = self.response_transform(torch.cat((v2_avg, v2_max), dim=1))
		# (b * K) x (hidden_size * 2)

		scores = torch.sum(torch.mul(v1, v2), dim=1).view(batch_size, -1)
		# b x K

		return scores