"""
Code from Modeling and Simulation in Python.

Copyright 2020 Allen Downey

MIT License: https://opensource.org/licenses/MIT
"""

import logging

logger = logging.getLogger(name="modsim.py")

# make sure we have Python 3.6 or better
import sys

if sys.version_info < (3, 6):
    logger.warning("modsim.py depends on Python 3.6 features.")

import inspect

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy

from scipy.interpolate import interp1d
from scipy.interpolate import InterpolatedUnivariateSpline

from scipy.integrate import odeint
from scipy.integrate import solve_ivp

from types import SimpleNamespace
from copy import copy

import pint

units = pint.UnitRegistry()
Quantity = units.Quantity

def flip(p=0.5):
    """Flips a coin with the given probability.

    p: float 0-1

    returns: boolean (True or False)
    """
    return np.random.random() < p

def cart2pol(x, y, z=None):
    """Convert Cartesian coordinates to polar.

    x: number or sequence
    y: number or sequence
    z: number or sequence (optional)

    returns: theta, rho OR theta, rho, z
    """
    x = np.asarray(x)
    y = np.asarray(y)

    rho = np.hypot(x, y)
    theta = np.arctan2(y, x)

    if z is None:
        return theta, rho
    else:
        return theta, rho, z

def pol2cart(theta, rho, z=None):
    """Convert polar coordinates to Cartesian.

    theta: number or sequence in radians
    rho: number or sequence
    z: number or sequence (optional)

    returns: x, y OR x, y, z
    """
    x = rho * np.cos(theta)
    y = rho * np.sin(theta)

    if z is None:
        return x, y
    else:
        return x, y, z

from numpy import linspace

def linrange(start, stop, step=1, **options):
    """Make an array of equally spaced values.

    start: first value
    stop: last value (might be approximate)
    step: difference between elements (should be consistent)

    returns: NumPy array
    """
    n = int(round((stop-start) / step))
    return linspace(start, stop, n+1, **options)

def leastsq(error_func, x0, *args, **options):
    """Find the parameters that yield the best fit for the data.

    `x0` can be a sequence, array, Series, or Params

    Positional arguments are passed along to `error_func`.

    Keyword arguments are passed to `scipy.optimize.leastsq`

    error_func: function that computes a sequence of errors
    x0: initial guess for the best parameters
    args: passed to error_func
    options: passed to leastsq

    :returns: Params object with best_params and ModSimSeries with details
    """
    # override `full_output` so we get a message if something goes wrong
    options["full_output"] = True

    # run leastsq
    t = scipy.optimize.leastsq(error_func, x0=x0, args=args, **options)
    best_params, cov_x, infodict, mesg, ier = t

    # pack the results into a ModSimSeries object
    details = ModSimSeries(infodict)
    details.set(cov_x=cov_x, mesg=mesg, ier=ier)

    # if we got a Params object, we should return a Params object
    if isinstance(x0, Params):
        best_params = Params(Series(best_params, x0.index))

    # return the best parameters and details
    return best_params, details

def minimize_scalar(min_func, bounds, *args, **options):
    """Finds the input value that minimizes `min_func`.

    Wrapper for
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.minimize_scalar.html

    min_func: computes the function to be minimized
    bounds: sequence of two values, lower and upper bounds of the range to be searched
    args: any additional positional arguments are passed to min_func
    options: any keyword arguments are passed as options to minimize_scalar

    returns: ModSimSeries object
    """
    try:
        min_func(bounds[0], *args)
    except Exception as e:
        msg = """Before running scipy.integrate.minimize_scalar, I tried
                 running the function you provided with the
                 lower bound, and I got the following error:"""
        logger.error(msg)
        raise (e)

    underride(options, xatol=1e-3)

    res = scipy.optimize.minimize_scalar(
        min_func,
        bracket=bounds,
        bounds=bounds,
        args=args,
        method="bounded",
        options=options,
    )

    if not res.success:
        msg = (
            """scipy.optimize.minimize_scalar did not succeed.
                 The message it returned is %s"""
            % res.message
        )
        raise Exception(msg)

    return res

def maximize_scalar(max_func, bounds, *args, **options):
    """Finds the input value that maximizes `max_func`.

    Wrapper for https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.minimize_scalar.html

    min_func: computes the function to be maximized
    bounds: sequence of two values, lower and upper bounds of the
            range to be searched
    args: any additional positional arguments are passed to max_func
    options: any keyword arguments are passed as options to minimize_scalar

    returns: ModSimSeries object
    """
    def min_func(*args):
        return -max_func(*args)

    res = minimize_scalar(min_func, bounds, *args, **options)

    # we have to negate the function value before returning res
    res.fun = -res.fun
    return res

def minimize_golden(min_func, bracket, *args, **options):
    """Find the minimum of a function by golden section search.

    Based on
    https://en.wikipedia.org/wiki/Golden-section_search#Iterative_algorithm

    :param min_func: function to be minimized
    :param bracket: interval containing a minimum
    :param args: arguments passes to min_func
    :param options: rtol and maxiter

    :return: ModSimSeries
    """
    maxiter = options.get("maxiter", 100)
    rtol = options.get("rtol", 1e-3)

    def success(**kwargs):
        return ModSimSeries(dict(success=True, **kwargs))

    def failure(**kwargs):
        return ModSimSeries(dict(success=False, **kwargs))

    a, b = bracket
    ya = min_func(a, *args)
    yb = min_func(b, *args)

    phi = 2 / (np.sqrt(5) - 1)
    h = b - a
    c = b - h / phi
    yc = min_func(c, *args)

    d = a + h / phi
    yd = min_func(d, *args)

    if yc > ya or yc > yb:
        return failure(message="The bracket is not well-formed.")

    for i in range(maxiter):

        # check for convergence
        if abs(h / c) < rtol:
            return success(x=c, fun=yc)

        if yc < yd:
            b, yb = d, yd
            d, yd = c, yc
            h = b - a
            c = b - h / phi
            yc = min_func(c, *args)
        else:
            a, ya = c, yc
            c, yc = d, yd
            h = b - a
            d = a + h / phi
            yd = min_func(d, *args)

    # if we exited the loop, too many iterations
    return failure(root=c, message="maximum iterations = %d exceeded" % maxiter)

def maximize_golden(max_func, bracket, *args, **options):
    """Find the maximum of a function by golden section search.

    :param min_func: function to be maximized
    :param bracket: interval containing a maximum
    :param args: arguments passes to min_func
    :param options: rtol and maxiter

    :return: ModSimSeries
    """

    def min_func(*args):
        return -max_func(*args)

    res = minimize_golden(min_func, bracket, *args, **options)

    # we have to negate the function value before returning res
    res.fun = -res.fun
    return res

def minimize_powell(min_func, x0, *args, **options):
    """Finds the input value that minimizes `min_func`.
    Wrapper for https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.minimize.html
    min_func: computes the function to be minimized
    x0: initial guess
    args: any additional positional arguments are passed to min_func
    options: any keyword arguments are passed as options to minimize_scalar
    returns: ModSimSeries object
    """
    underride(options, tol=1e-3)

    res = scipy.optimize.minimize(min_func, x0, *args, **options)

    return ModSimSeries(res)

# make aliases for minimize and maximize
minimize = minimize_golden
maximize = maximize_golden

def run_solve_ivp(system, slope_func, **options):
    """Computes a numerical solution to a differential equation.

    `system` must contain `init` with initial conditions,
    `t_0` with the start time, and `t_end` with the end time.

    It can contain any other parameters required by the slope function.

    `options` can be any legal options of `scipy.integrate.solve_ivp`

    system: System object
    slope_func: function that computes slopes

    returns: TimeFrame
    """
    system = remove_units(system)

    # make sure `system` contains `init`
    if not hasattr(system, "init"):
        msg = """It looks like `system` does not contain `init`
                 as a system variable.  `init` should be a State
                 object that specifies the initial condition:"""
        raise ValueError(msg)

    # make sure `system` contains `t_end`
    if not hasattr(system, "t_end"):
        msg = """It looks like `system` does not contain `t_end`
                 as a system variable.  `t_end` should be the
                 final time:"""
        raise ValueError(msg)

    # the default value for t_0 is 0
    t_0 = getattr(system, "t_0", 0)

    # try running the slope function with the initial conditions
    try:
        slope_func(t_0, system.init, system)
    except Exception as e:
        msg = """Before running scipy.integrate.solve_ivp, I tried
                 running the slope function you provided with the
                 initial conditions in `system` and `t=t_0` and I got
                 the following error:"""
        logger.error(msg)
        raise (e)

    # get the list of event functions
    events = options.get('events', [])

    # if there's only one event function, put it in a list
    try:
        iter(events)
    except TypeError:
        events = [events]

    for event_func in events:
        # make events terminal unless otherwise specified
        if not hasattr(event_func, 'terminal'):
            event_func.terminal = True

        # test the event function with the initial conditions
        try:
            event_func(t_0, system.init, system)
        except Exception as e:
            msg = """Before running scipy.integrate.solve_ivp, I tried
                     running the event function you provided with the
                     initial conditions in `system` and `t=t_0` and I got
                     the following error:"""
            logger.error(msg)
            raise (e)

    # get dense output unless otherwise specified
    underride(options, dense_output=True)

    # run the solver
    bunch = solve_ivp(slope_func, [t_0, system.t_end], system.init,
                      args=[system], **options)

    # separate the results from the details
    y = bunch.pop("y")
    t = bunch.pop("t")

    # get the column names from `init`, if possible
    if hasattr(system.init, 'index'):
        columns = system.init.index
    else:
        columns = range(len(system.init))

    # evaluate the results at equally-spaced points
    if options.get('dense_output', False):
        try:
            num = system.num
        except AttributeError:
            num = 51
        t_final = t[-1]
        t_array = linspace(t_0, t_final, num)
        y_array = bunch.sol(t_array)

        # pack the results into a TimeFrame
        results = TimeFrame(y_array.T, index=t_array,
                        columns=columns)
    else:
        results = TimeFrame(y.T, index=t,
                        columns=columns)

    return results, bunch

def check_system(system, slope_func):
    """Make sure the system object has the fields we need for run_ode_solver.

    :param system:
    :param slope_func:
    :return:
    """
    # make sure `system` contains `init`
    if not hasattr(system, "init"):
        msg = """It looks like `system` does not contain `init`
                 as a system variable.  `init` should be a State
                 object that specifies the initial condition:"""
        raise ValueError(msg)

    # make sure `system` contains `t_end`
    if not hasattr(system, "t_end"):
        msg = """It looks like `system` does not contain `t_end`
                 as a system variable.  `t_end` should be the
                 final time:"""
        raise ValueError(msg)

    # the default value for t_0 is 0
    t_0 = getattr(system, "t_0", 0)

    # get the initial conditions
    init = system.init

    # get t_end
    t_end = system.t_end

    # if dt is not specified, take 100 steps
    try:
        dt = system.dt
    except AttributeError:
        dt = t_end / 100

    return init, t_0, t_end, dt

def run_euler(system, slope_func, **options):
    """Computes a numerical solution to a differential equation.

    `system` must contain `init` with initial conditions,
    `t_end` with the end time, and `dt` with the time step.

    `system` may contain `t_0` to override the default, 0

    It can contain any other parameters required by the slope function.

    `options` can be ...

    system: System object
    slope_func: function that computes slopes

    returns: TimeFrame
    """
    # the default message if nothing changes
    msg = "The solver successfully reached the end of the integration interval."

    # get parameters from system
    init, t_0, t_end, dt = check_system(system, slope_func)

    # make the TimeFrame
    frame = TimeFrame(columns=init.index)
    frame.row[t_0] = init
    ts = linrange(t_0, t_end, dt) * get_units(t_end)

    # run the solver
    for t1 in ts:
        y1 = frame.row[t1]
        slopes = slope_func(y1, t1, system)
        y2 = [y + slope * dt for y, slope in zip(y1, slopes)]
        t2 = t1 + dt
        frame.row[t2] = y2

    details = ModSimSeries(dict(message="Success"))
    return frame, details

def run_ralston(system, slope_func, **options):
    """Computes a numerical solution to a differential equation.

    `system` must contain `init` with initial conditions,
     and `t_end` with the end time.

     `system` may contain `t_0` to override the default, 0

    It can contain any other parameters required by the slope function.

    `options` can be ...

    system: System object
    slope_func: function that computes slopes

    returns: TimeFrame
    """
    # the default message if nothing changes
    msg = "The solver successfully reached the end of the integration interval."

    # get parameters from system
    init, t_0, t_end, dt = check_system(system, slope_func)

    # make the TimeFrame
    frame = TimeFrame(columns=init.index)
    frame.row[t_0] = init
    ts = linrange(t_0, t_end, dt) * get_units(t_end)

    event_func = options.get("events", None)
    z1 = np.nan

    def project(y1, t1, slopes, dt):
        t2 = t1 + dt
        y2 = [y + slope * dt for y, slope in zip(y1, slopes)]
        return y2, t2

    # run the solver
    for t1 in ts:
        y1 = frame.row[t1]

        # evaluate the slopes at the start of the time step
        slopes1 = slope_func(y1, t1, system)

        # evaluate the slopes at the two-thirds point
        y_mid, t_mid = project(y1, t1, slopes1, 2 * dt / 3)
        slopes2 = slope_func(y_mid, t_mid, system)

        # compute the weighted sum of the slopes
        slopes = [(k1 + 3 * k2) / 4 for k1, k2 in zip(slopes1, slopes2)]

        # compute the next time stamp
        y2, t2 = project(y1, t1, slopes, dt)

        # check for a terminating event
        if event_func:
            z2 = event_func(y2, t2, system)
            if z1 * z2 < 0:
                scale = magnitude(z1 / (z1 - z2))
                y2, t2 = project(y1, t1, slopes, scale * dt)
                frame.row[t2] = y2
                msg = "A termination event occurred."
                break
            else:
                z1 = z2

        # store the results
        frame.row[t2] = y2

    details = ModSimSeries(dict(success=True, message=msg))
    return frame, details

run_ode_solver = run_ralston

# TODO: Implement leapfrog

def fsolve(func, x0, *args, **options):
    """Return the roots of the (non-linear) equations
    defined by func(x) = 0 given a starting estimate.

    Uses scipy.optimize.fsolve, with extra error-checking.

    func: function to find the roots of
    x0: scalar or array, initial guess
    args: additional positional arguments are passed along to fsolve,
          which passes them along to func

    returns: solution as an array
    """
    # make sure we can run the given function with x0
    try:
        func(x0, *args)
    except Exception as e:
        msg = """Before running scipy.optimize.fsolve, I tried
                 running the error function you provided with the x0
                 you provided, and I got the following error:"""
        logger.error(msg)
        raise (e)

    # make the tolerance more forgiving than the default
    underride(options, xtol=1e-6)

    # run fsolve
    result = scipy.optimize.fsolve(func, x0, args=args, **options)

    return result

def crossings(series, value):
    """Find the labels where the series passes through value.

    The labels in series must be increasing numerical values.

    series: Series
    value: number

    returns: sequence of labels
    """
    values = series.values - value
    interp = InterpolatedUnivariateSpline(series.index, values)
    return interp.roots()

def has_nan(a):
    """Checks whether the an array contains any NaNs.

    :param a: NumPy array or Pandas Series
    :return: boolean
    """
    return np.any(np.isnan(a))

def is_strictly_increasing(a):
    """Checks whether the elements of an array are strictly increasing.

    :param a: NumPy array or Pandas Series
    :return: boolean
    """
    return np.all(np.diff(a) > 0)

def interpolate(series, **options):
    """Creates an interpolation function.

    series: Series object
    options: any legal options to scipy.interpolate.interp1d

    returns: function that maps from the index to the values
    """
    if has_nan(series.index):
        msg = """The Series you passed to interpolate contains
                 NaN values in the index, which would result in
                 undefined behavior.  So I'm putting a stop to that."""
        raise ValueError(msg)

    if not is_strictly_increasing(series.index):
        msg = """The Series you passed to interpolate has an index
                 that is not strictly increasing, which would result in
                 undefined behavior.  So I'm putting a stop to that."""
        raise ValueError(msg)

    # make the interpolate function extrapolate past the ends of
    # the range, unless `options` already specifies a value for `fill_value`
    underride(options, fill_value="extrapolate")

    # call interp1d, which returns a new function object
    x = series.index
    y = series.values
    interp_func = interp1d(x, y, **options)
    return interp_func

def interpolate_inverse(series, **options):
    """Interpolate the inverse function of a Series.

    series: Series object, represents a mapping from `a` to `b`
    options: any legal options to scipy.interpolate.interp1d

    returns: interpolation object, can be used as a function
             from `b` to `a`
    """
    inverse = Series(series.index, index=series.values)
    interp_func = interpolate(inverse, **options)
    return interp_func

def gradient(series, **options):
    """Computes the numerical derivative of a series.

    If the elements of series have units, they are dropped.

    series: Series object
    options: any legal options to np.gradient

    returns: Series, same subclass as series
    """
    x = series.index
    y = series.values

    a = np.gradient(y, x, **options)
    return series.__class__(a, series.index)

def source_code(obj):
    """Prints the source code for a given object.

    obj: function or method object
    """
    print(inspect.getsource(obj))

def underride(d, **options):
    """Add key-value pairs to d only if key is not in d.

    If d is None, create a new dictionary.

    d: dictionary
    options: keyword args to add to d
    """
    if d is None:
        d = {}

    for key, val in options.items():
        d.setdefault(key, val)

    return d

def contour(df, **options):
    """Makes a contour plot from a DataFrame.

    Wrapper for plt.contour
    https://matplotlib.org/3.1.0/api/_as_gen/matplotlib.pyplot.contour.html

    Note: columns and index must be numerical

    df: DataFrame
    options: passed to plt.contour
    """
    fontsize = options.pop("fontsize", 12)
    underride(options, cmap="viridis")
    x = df.columns
    y = df.index
    X, Y = np.meshgrid(x, y)
    cs = plt.contour(X, Y, df, **options)
    plt.clabel(cs, inline=1, fontsize=fontsize)

def savefig(filename, **options):
    """Save the current figure.

    Keyword arguments are passed along to plt.savefig

    https://matplotlib.org/api/_as_gen/matplotlib.pyplot.savefig.html

    filename: string
    """
    print("Saving figure to file", filename)
    plt.savefig(filename, **options)

def decorate(**options):
    """Decorate the current axes.

    Call decorate with keyword arguments like
    decorate(title='Title',
             xlabel='x',
             ylabel='y')

    The keyword arguments can be any of the axis properties
    https://matplotlib.org/api/axes_api.html
    """
    ax = plt.gca()
    ax.set(**options)

    handles, labels = ax.get_legend_handles_labels()
    if handles:
        ax.legend(handles, labels)

    plt.tight_layout()

def remove_from_legend(bad_labels):
    """Removes some labels from the legend.

    bad_labels: sequence of strings
    """
    ax = plt.gca()
    handles, labels = ax.get_legend_handles_labels()
    handle_list, label_list = [], []
    for handle, label in zip(handles, labels):
        if label not in bad_labels:
            handle_list.append(handle)
            label_list.append(label)
    ax.legend(handle_list, label_list)

class SettableNamespace(SimpleNamespace):
    """Contains a collection of parameters.

    Used to make a System object.

    Takes keyword arguments and stores them as attributes.
    """
    def __init__(self, namespace=None, **kwargs):
        super().__init__()
        if namespace:
            self.__dict__.update(namespace.__dict__)
        self.__dict__.update(kwargs)

    def get(self, name, default=None):
        """Look up a variable.

        name: string varname
        default: value returned if `name` is not present
        """
        try:
            return self.__getattribute__(name, default)
        except AttributeError:
            return default

    def set(self, **variables):
        """Make a copy and update the given variables.

        returns: Params
        """
        new = copy(self)
        new.__dict__.update(variables)
        return new

def magnitude(x):
    """Returns the magnitude of a Quantity or number.

    x: Quantity or number

    returns: number
    """
    return x.magnitude if hasattr(x, 'magnitude') else x

def remove_units(namespace):
    """Removes units from the values in a Namespace.

    Only removes units from top-level values;
    does not traverse nested values.

    returns: new Namespace object
    """
    res = copy(namespace)
    for label, value in res.__dict__.items():
        if isinstance(value, pd.Series):
            value = remove_units_series(value)
        res.__dict__[label] = magnitude(value)
    return res

def remove_units_series(series):
    """Removes units from the values in a Series.

    Only removes units from top-level values;
    does not traverse nested values.

    returns: new Series object
    """
    res = copy(series)
    for label, value in res.iteritems():
        res[label] = magnitude(value)
    return res

class System(SettableNamespace):
    """Contains system parameters and their values.

    Takes keyword arguments and stores them as attributes.
    """
    pass

class Params(SettableNamespace):
    """Contains system parameters and their values.

    Takes keyword arguments and stores them as attributes.
    """
    pass

def State(**variables):
    """Contains the values of state variables."""
    return pd.Series(variables)

def TimeSeries(*args, **kwargs):
    """
    """
    if args or kwargs:
        series = pd.Series(*args, **kwargs)
    else:
        series = pd.Series([], dtype=np.float64)

    series.index.name = 'Time'
    if 'name' not in kwargs:
        series.name = 'Quantity'
    return series

def SweepSeries(*args, **kwargs):
    """
    """
    if args or kwargs:
        series = pd.Series(*args, **kwargs)
    else:
        series = pd.Series([], dtype=np.float64)

    series.index.name = 'Parameter'
    if 'name' not in kwargs:
        series.name = 'Metric'
    return series

def TimeFrame(*args, **kwargs):
    """DataFrame that maps from time to State.
    """
    return pd.DataFrame(*args, **kwargs)

def SweepFrame(*args, **kwargs):
    """DataFrame that maps from parameter value to SweepSeries.
    """
    return pd.DataFrame(*args, **kwargs)

def Vector(x, y, z=None, **options):
    """
    """
    if z is None:
        return pd.Series(dict(x=x, y=y), **options)
    else:
        return pd.Series(dict(x=x, y=y, z=z), **options)

## Vector functions (should work with any sequence)

def vector_mag(v):
    """Vector magnitude."""
    return np.sqrt(np.dot(v, v))

def vector_mag2(v):
    """Vector magnitude squared."""
    return np.dot(v, v)

def vector_angle(v):
    """Angle between v and the positive x axis.

    Only works with 2-D vectors.

    returns: angle in radians
    """
    assert len(v) == 2
    x, y = v
    return np.arctan2(y, x)

def vector_polar(v):
    """Vector magnitude and angle.

    returns: (number, angle in radians)
    """
    return vector_mag(v), vector_angle(v)

def vector_hat(v):
    """Unit vector in the direction of v.

    returns: Vector or array
    """
    # check if the magnitude of the Quantity is 0
    mag = vector_mag(v)
    if mag == 0:
        return v
    else:
        return v / mag

def vector_perp(v):
    """Perpendicular Vector (rotated left).

    Only works with 2-D Vectors.

    returns: Vector
    """
    assert len(v) == 2
    x, y = v
    return Vector(-y, x)

def vector_dot(v, w):
    """Dot product of v and w.

    returns: number or Quantity
    """
    return np.dot(v, w)

def vector_cross(v, w):
    """Cross product of v and w.

    returns: number or Quantity for 2-D, Vector for 3-D
    """
    res = np.cross(v, w)

    if len(v) == 3:
        return Vector(*res)
    else:
        return res

def vector_proj(v, w):
    """Projection of v onto w.

    returns: array or Vector with direction of w and units of v.
    """
    w_hat = vector_hat(w)
    return vector_dot(v, w_hat) * w_hat

def scalar_proj(v, w):
    """Returns the scalar projection of v onto w.

    Which is the magnitude of the projection of v onto w.

    returns: scalar with units of v.
    """
    return vector_dot(v, vector_hat(w))

def vector_dist(v, w):
    """Euclidean distance from v to w, with units."""
    if isinstance(v, list):
        v = np.asarray(v)
    return vector_mag(v - w)

def vector_diff_angle(v, w):
    """Angular difference between two vectors, in radians.
    """
    if len(v) == 2:
        return vector_angle(v) - vector_angle(w)
    else:
        # TODO: see http://www.euclideanspace.com/maths/algebra/
        # vectors/angleBetween/
        raise NotImplementedError()

def plot_segment(A, B, **options):
    """Plots a line segment between two Vectors.

    For 3-D vectors, the z axis is ignored.

    Additional options are passed along to plot().

    A: Vector
    B: Vector
    """
    xs = A.x, B.x
    ys = A.y, B.y
    plot(xs, ys, **options)

from time import sleep
from IPython.display import clear_output

def animate(results, draw_func, *args, interval=None):
    """Animate results from a simulation.

    results: TimeFrame
    draw_func: function that draws state
    interval: time between frames in seconds
    """
    plt.figure()
    try:
        for t, state in results.iterrows():
            draw_func(t, state, *args)
            plt.show()
            if interval:
                sleep(interval)
            clear_output(wait=True)
        draw_func(t, state, *args)
        plt.show()
    except KeyboardInterrupt:
        pass


from PIL import Image
import boto3
import time
import os
import json

base = os.getcwd()
manPhotoDir = os.path.join(base,'ver_3')
#photo = '3.png'
region='eu-west-1'

#with open(photo, 'rb') as source_image:
#    source_bytes = source_image.read()
xVectors = [-50, 0, 50]
yVectors = [100, 150, 200, 250, 300]
savedPhotos = [1, 2, 3, 4, 5, 6]
photosType = ['FrontOff1', 'FrontOff2', 'FrontOn1', 'FrontOn2', 'Back1', 'Back2']
fileNames =list()
print('geting paths of files')
for y in yVectors:
    for x in xVectors:
        for photoNo in savedPhotos:
            fileNames.append(os.path.join(os.path.join(manPhotoDir, str(y) + '_' + str(x) ), str(photoNo) +'.bmp'))
resultDictionary = dict.fromkeys(fileNames , '')
formatTimes = list()
print('convert bmps to png')
for fName in fileNames:
    start_time = time.time()
    Image.open(fName).save(fName.replace('.bmp', '.png'))
    resultDictionary[fName]=str((time.time() - start_time)*1000)
#    formatTimes.append()
#print("Times for coverting files")
#print(', '.join(str(x) for x in formatTimes))
print('sending requests')
client = boto3.client('rekognition')
for fName in fileNames:
    with open(fName.replace('.bmp', '.png'), 'rb') as source_image:
        source_bytes = source_image.read()
    start_time = time.time()
    response = client.detect_labels(Image={'Bytes': source_bytes}, MaxLabels = 3)
    resultDictionary[fName]+=';'+str((time.time() - start_time)*1000)+os.linesep+json.dumps(response)
print('saving responses')
a_file = open('results.json', 'w')
json.dump(resultDictionary, a_file)
a_file.close()


import datetime

from django.core.exceptions import ValidationError
from django.forms import ModelForm, ModelMultipleChoiceField, forms, CheckboxSelectMultiple, DateInput, TextInput, \
    BooleanField
from django.db.models import Prefetch
from django.template import defaultfilters

from urnik.models import Oseba, Predmet, Ucilnica, Rezervacija
from urnik.templatetags.tags import dan_tozilnik_mnozina

class PredmetModelMultipleChoiceField(ModelMultipleChoiceField):
    def label_from_instance(self, obj):
        return obj.opisno_ime()

class OsebeModelMultipleChoiceField(ModelMultipleChoiceField):
    def label_from_instance(self, obj):
        return obj.priimek_ime

class UcilniceModelMultipleChoiceField(ModelMultipleChoiceField):
    def label_from_instance(self, obj):
        return obj.oznaka

class KombiniranPogledForm(forms.Form):
    oseba = OsebeModelMultipleChoiceField(queryset=Oseba.objects.aktivni(), required=False)
    predmet = PredmetModelMultipleChoiceField(queryset=Predmet.objects.exclude(ime="").prefetch_related('letniki'), required=False)

class RezevacijeForm(ModelForm):

    PREKRIVANJA = 'prekrivanja'

    osebe = OsebeModelMultipleChoiceField(queryset=Oseba.objects.all(),
                                          help_text='Osebe, ki si lastijo to rezervacijo. Te osebe bodo lahko rezervacijo '
                                                    'spreminjale, prikazala pa se bo tudi na njihovem tedenskem urniku.')
    ucilnice = UcilniceModelMultipleChoiceField(queryset=Ucilnica.objects.objavljene(),
                                                help_text='Izberite učilnice, ki jih želite rezervirati',
                                                widget=CheckboxSelectMultiple())
    predmeti = PredmetModelMultipleChoiceField(queryset=Predmet.objects.all(),
                                               required=False,
                                               help_text='Predmeti, povezani s to rezervacijo. Lahko pustite prazno. '
                                                         'Če izberete enega ali več predmetov, se bo rezervacija pokazala na tedenskem '
                                                         'urniku predmeta in vseh letnikov, ki predmet imajo.')
    use_required_attribute = False

    class Meta:
        model = Rezervacija
        fields = ['ucilnice', 'osebe', 'dan', 'dan_konca', 'od', 'do', 'opomba', 'predmeti']
        widgets = {
            'dan': DateInput(attrs={'placeholder': 'npr. 15. 1. 2019', 'class': 'datepicker'}),
            'dan_konca': DateInput(attrs={'placeholder': 'ponavadi prazno, lahko tudi npr. 17. 1. 2019', 'class': 'datepicker'}),
            'opomba': TextInput(attrs={'placeholder': 'npr. "sestanek raziskovalne skupine", ali pa "kolokvij ALG1"'}),
        }

    def __init__(self, *args, **kwargs):
        dovoli_prekrivanja = kwargs.pop('dovoli_prekrivanja', None)
        super(RezevacijeForm, self).__init__(*args, **kwargs)
        if dovoli_prekrivanja:
            self.fields['ignoriraj_prekrivanja'] = BooleanField(
                required=False, initial=False,
                help_text='Zavedam se prekrivanj in vseeno želim rezervirati izbrane učilnice '
                          'v izbranih dnevih in urah.')

    def clean_dan(self):
        dan = self.cleaned_data['dan']
        if dan < datetime.date.today():
            raise ValidationError("Datum rezervacije mora biti v prihodnosti.")
        return dan

    def clean(self):
        cleaned = super().clean()
        if self.errors:
            return cleaned

        od = cleaned.get('od')
        do = cleaned.get('do')
        if od >= do:
            self.add_error(None, ValidationError("Ura začetka rezervacije mora biti pred uro konca rezervacije."))

        dan = cleaned.get('dan')
        konec = cleaned.get('dan_konca')
        if konec:
            if dan == konec:
                self.cleaned_data['dan_konca'] = None
            elif dan > konec:
                self.add_error(None, ValidationError("Dan začetka rezervacije moda biti "
                                                     "pred dnevom konca rezervacije."))
        ignoriraj = cleaned.get('ignoriraj_prekrivanja')
        if not ignoriraj:
            self._preveri_konflikte(cleaned)

        return self.cleaned_data

    def _preveri_konflikte(self, cleaned):
        from urnik.iskanik_konfliktov import IskalnikKonfliktov
        ucilnice = cleaned.get('ucilnice')
        dan = cleaned.get('dan')
        konec = cleaned.get('dan_konca') or dan
        od = cleaned.get('od')
        do = cleaned.get('do')
        iskalnik = IskalnikKonfliktov(ucilnice, dan, konec)
        iskalnik.dodaj_srecanja()
        iskalnik.dodaj_rezervacije(Rezervacija.objects.prihajajoce().prefetch_related(Prefetch('ucilnice', to_attr='seznam_ucilnic')))

        date_format = lambda d: defaultfilters.date(d, "D, j. b")
        for u in ucilnice:
            if u.dovoli_veckratno_rezervacijo:
                continue
            d = dan
            while d <= konec:
                konflikti = iskalnik.konflikti(u, d, od, do)
                for r in konflikti.rezervacije:
                    oseba = r.osebe.all()[:1]
                    self.add_error(None, ValidationError(
                        'Vaša rezervacija bi se prekrivala z rezervacijo osebe %(oseba)s %(dan)s '
                        'od %(od)i do %(do)i z razlogom %(razlog)s.',
                        params={
                            'oseba': oseba[0] if oseba else 'neznan',
                            'dan': "od {} do {}".format(date_format(r.dan), date_format(r.dan_konca))
                            if r.dan_konca else "dne {}".format(date_format(r.dan)),
                            'razlog': r.opomba,
                            'od': r.od,
                            'do': r.do,
                        },
                        code=RezevacijeForm.PREKRIVANJA,
                    ))

                for s in konflikti.srecanja:
                    self.add_error(None, ValidationError(
                        'Vaša rezervacija bi se prekrivala s predmetom %(predmet)s (%(semester)s), ki se izvaja '
                        'ob %(dan_v_tednu)s od %(od)i do %(do)i.',
                        params={
                            'predmet': s.predmet,
                            'semester': s.semester,
                            'dan_v_tednu': dan_tozilnik_mnozina(s.dan),
                            'od': s.od,
                            'do': s.do,
                        },
                        code=RezevacijeForm.PREKRIVANJA,
                    ))

                d += datetime.timedelta(days=1)


import torch.nn as nn

def cos_loss(x, y):
    return 2 - (x).mul(y).mean()

def l1_loss(x, y):
    return (x - y).abs().mean()

def l2_loss(x, y, age=True):
    if age:  # 1/(M*N)
        loss = (x - y).pow(2)
    else:  # 1/N
        loss = (x - y).pow(2).view(list(x.size())[0], -1).sum(dim=-1)

    return loss.mean()

class KL_Loss_AGE(nn.Module):
    """
    Kullback–Leibler Loss for the AGE model
    """

    def __init__(self, minimize):
        """

        :param minimize: Boolean parameter to change between minimizing and maximizing the Loss.
        """
        super(KL_Loss_AGE, self).__init__()
        self.minimize = minimize
        self.mean = 0
        self.var = 0
        self.M = 0

    def forward(self, z):
        # Input normalized z
        """
        :param z: The input/output (depends on the occasion) in the latent dimensional space
        :return: The loss of the data
        """
        self.M = list(z.size())[1]  # size of latent space
        self.mean = z.mean(dim=0)
        self.var = z.var(dim=0, unbiased=False)
        kl_loss = -1 / 2 + ((self.mean.pow(2) + self.var) / 2 - self.var.sqrt().log()).mean()

        if not self.minimize:
            kl_loss *= -1

        return kl_loss

class KL_Loss_Intro(nn.Module):
    """
       Kullback–Leibler Loss for the IntroVAE model
    """

    def __init__(self, minimize):
        """

        :param minimize: Boolean parameter to change between minimizing and maximizing the Loss.

        """
        super(KL_Loss_Intro, self).__init__()
        self.minimize = minimize
        self.mean = 0
        self.var = 0
        self.M = 0
        self.N = 0

    def forward(self, mean, logvar):
        """

        :param mean: The mean of the input in the latent space
        :param logvar: The variance of the input in the latent space
        :return:
        """

        # Input mean and variance of z
        self.M, self.N = list(mean.size())
        self.mean = mean
        self.logvar = logvar
        var = logvar.exp()
        kl_loss = (-1 - self.logvar + self.mean.pow(2) + var).mul_(0.5).sum(dim=-1)
        kl_loss = kl_loss.mean()

        if not self.minimize:
            kl_loss *= -1

        return kl_loss


#  Licensed under the Apache License, Version 2.0 (the "License"); you may
#  not use this file except in compliance with the License. You may obtain
#  a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#  License for the specific language governing permissions and limitations
#  under the License.
#
#  COPYRIGHT  (C)  NEC  CORPORATION  2016

import logging

from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _

from horizon import messages
from horizon import tables
from openstack_dashboard.api import base

from nec_portal.api.cloudify import Cloudify as cloudify_api

LOG = logging.getLogger(__name__)

class ExecuteInstall(tables.BatchAction):
    """Execute install class"""
    name = 'execute_install'
    verbose_name = _('Execute Install')
    policy_rules = (("tosca", "tosca:execution_install"), )
    classes = ('tosca_confirm',)

    @staticmethod
    def action_present(count):
        return _('Execute Install')

    @staticmethod
    def action_past(count):
        return _('Executed Install')

    def action(self, request, object_id):
        try:
            cloudify_api(
                url_for_orchestration=(
                    base.url_for(request, 'orchestration')
                ),
                token_id=request.user.token.id
            ).do_install(object_id)
        except Exception as e:
            LOG.error('DOSCA: Failed to start install. (%s)' % e)
            messages.error(request, e.message)
            raise Exception()

class ExecuteUninstall(tables.DeleteAction):
    """Execute uninstall class"""
    name = 'execute_uninstall'
    verbose_name = _('Execute Uninstall')
    policy_rules = (("tosca", "tosca:execution_uninstall"), )
    classes = ('btn-danger',)

    @staticmethod
    def action_present(count):
        return _('Execute Uninstall')

    @staticmethod
    def action_past(count):
        return _('Executed Uninstall')

    def delete(self, request, object_id):
        try:
            cloudify_api(
                url_for_orchestration=(
                    base.url_for(request, 'orchestration')
                ),
                token_id=request.user.token.id
            ).do_uninstall(object_id)
        except Exception as e:
            LOG.error('DOSCA: Failed to uninstall. (%s)' % e)
            messages.error(request, e.message)
            raise Exception()

class DeleteDeployment(tables.DeleteAction):
    """Delete deployment class"""
    name = 'delete_deployment'
    success_url = 'horizon:project:tosca_deployments:index'
    policy_rules = (("tosca", "tosca:deployment_delete"), )

    @staticmethod
    def action_present(count):
        return _('Delete Deployment')

    @staticmethod
    def action_past(count):
        return _('Delete Deployment')

    def delete(self, request, object_id):
        try:
            cloudify_api(
                url_for_orchestration=(
                    base.url_for(request, 'orchestration')
                ),
                token_id=request.user.token.id
            ).delete_deployment(object_id)
        except Exception as e:
            LOG.error('DOSCA: Failed to delete demployment. (%s)' % e)
            messages.error(request, e.message)
            raise Exception()

class CancelExecution(tables.DeleteAction):
    """Cancel execution class"""
    name = 'cancel_execution'
    verbose_name = _('Cancel Execution')
    force = False

    @staticmethod
    def action_present(count):
        return _('Cancel Execution')

    @staticmethod
    def action_past(count):
        return _('Canceled Execution')

    def delete(self, request, object_id):
        try:
            cloudify_api(
                url_for_orchestration=(
                    base.url_for(request, 'orchestration')
                ),
                token_id=request.user.token.id
            ).cancel_execution(object_id, self.force)
        except Exception as e:
            LOG.error('DOSCA: Failed to cancel. (%s)' % e)
            messages.error(request, e.message)
            raise Exception()

    def allowed(self, request, object):
        if object.status:
            if object.status == 'cancelling':
                self.force = True
            if object.status in ['started', 'cancelling']:
                return True
        return False

def get_execution_link_url(deployment):
    url = 'horizon:project:tosca_deployments:executions'
    return reverse(url, args=[deployment.id])

def get_eventlog_link_url(execution):
    url = 'horizon:project:tosca_deployments:eventlog'
    return reverse(url, args=[execution.id])

class ToscaDeploymentTable(tables.DataTable):
    """Tosca Deployment Table class"""
    deployment_name = tables.Column('id',
                                    verbose_name=_('Deployment Name'),
                                    link=get_execution_link_url)
    template_name = tables.Column('template_id',
                                  verbose_name=_('Template Name'))
    created_at = tables.Column('created_at',
                               verbose_name=_('Created At'))

    def get_object_id(self, deployment):
        return deployment.id

    class Meta(object):
        """Meta class"""
        name = 'tosca_deployment_list'
        verbose_name = 'Deployments'

        row_actions = (ExecuteInstall, ExecuteUninstall,
                       DeleteDeployment,)
        multi_select = False

class ToscaExecutionTable(tables.DataTable):
    """Tosca Execution Table class"""
    id = tables.Column('id',
                       verbose_name=_('Execution ID'),
                       link=get_eventlog_link_url,
                       sortable=False)
    workflow_id = tables.Column('workflow_id',
                                verbose_name=_('Workflow ID'),
                                sortable=False)
    status = tables.Column('status',
                           verbose_name=_('Status'),
                           sortable=False)
    error = tables.Column('error',
                          verbose_name=_('Error'),
                          sortable=False)
    created_at = tables.Column('created_at',
                               verbose_name=_('Created At'),
                               sortable=False)

    def get_object_id(self, execution):
        return execution.id

    class Meta(object):
        """Meta class"""
        name = 'tosca_deployment_list'
        verbose_name = 'Deployments'

        row_actions = (CancelExecution,)
        multi_select = False


from __future__ import print_function

import os
import re

import burlap
from burlap import ContainerSatchel
from burlap.constants import *
from burlap.decorators import task

fabfile_template = os.path.join(
    os.path.dirname(burlap.__file__),
    'templates',
    'burlap',
    'fabfile.py.template',
)

def md(d):
    if os.path.isdir(d):
        return
    os.makedirs(d)

def to_camelcase(value):
    value = re.sub(r'[^a-zA-Z0-9]+', ' ', value).strip()
    return ''.join(x.capitalize() for x in value.split(' '))

def init_dj(project_name, default_roles, virtualenv_dir='.env', version=None, **kwargs):

    site_name = project_name

    print('Installing Django...')
    if version:
        os.system('%s/bin/pip install Django==%s' % (virtualenv_dir, version))
    else:
        os.system('%s/bin/pip install Django' % virtualenv_dir)

    print('Initializing Django project...')
    if not os.path.isdir('src/%s' % site_name):
        print('Initializing base django project...')
        os.system('. %s/bin/activate; django-admin.py startproject %s src; deactivate' % (virtualenv_dir, site_name,))
        _settings_fn = os.path.abspath('src/%s/settings.py' % project_name)
        _content = open(_settings_fn, 'r').read()
        _sites = '''SITE_{name_upper} = "{name_lower}"
SITES = (
    SITE_{name_upper},
)
'''.format(
            name_upper=project_name.upper(),
            name_lower=project_name.lower(),
        )
        _top = []
        for _role in default_roles:
            _top.append("ROLE_%s = '%s'" % (_role.upper(), _role.lower()))
        _top.append('ROLES = (')
        for _role in default_roles:
            _top.append("    ROLE_%s," % (_role.upper(),))
        _top.append(')')
        _index = _content.find('"""\n\n')+4

        bottom_args = dict(
            app_name=project_name,
            app_name_title=project_name.title() + ' Administration',
            app_name_simple=project_name.title()
        )
        _bottom = '''
PROJECT_DIR = os.path.abspath(os.path.join(os.path.split(__file__)[0], '..', '..'))

STATIC_ROOT = os.path.join(PROJECT_DIR, 'static')

MEDIA_ROOT = os.path.join(PROJECT_DIR, 'media')
MEDIA_URL = '/media/'

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)
TEMPLATE_DIRS = (
    '%s/src/{app_name}/templates' % PROJECT_DIR,
)
# https://docs.djangoproject.com/en/1.11/ref/settings/#templates
TEMPLATES = [
    {{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': TEMPLATE_DIRS,
        'APP_DIRS': True,
        'OPTIONS': {{
            #'loaders': TEMPLATE_LOADERS, # Unnecessary if we're using APP_DIRS.
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        }},
    }},
]
ADMIN_TITLE = '{app_name_title}'
ADMIN_TITLE_SIMPLE = '{app_name_simple}'
'''.format(**bottom_args)
        open(_settings_fn, 'w').write(_content[:_index]+_sites+('\n'.join(_top))+_content[_index:]+_bottom)

    print('Creating Django helper scripts...')
    open('src/manage', 'w').write('''#!/bin/bash
# Helper script for ensuring we use the Python binary in our local
# virtual environment when calling management commands.
# Otherwise, we'd have to always run `. ../.env/bin/activate`, which can be
# annoying.
# Be sue to run `fab <role> pip.init` first in order to setup
# the target role's Python virtual environment.
DIR=`dirname $0`;
cd $DIR;
../.env/bin/python manage.py $@''')
    open('src/runserver', 'w').write('''#!/bin/bash
# Helper script for running the local dev server, ensuring
# our virtual environment is used.
#set -e
#script_dir=`dirname $0`
#cd $script_dir
if [ -z "$PORT" ]; then
export PORT=8111
fi
if [ -z "$ROLE" ]; then
export ROLE=dev
fi
. ~/.bash_aliases
./manage runserver localhost:$PORT''')
    open('src/shell', 'w').write(r'''#!/bin/bash
# Creates a local PIP-aware shell.
#set -e
if [ $_ == $0 ]
then
echo "Please source this script. Do not execute."
exit 1
fi
#script_dir=`dirname $0`
#cd $script_dir
. .env/bin/activate
PS1="\u@\h:\W(fab)\$ "''')

    md('media')
    md('static')

    os.system('chmod +x src/shell')
    os.system('chmod +x src/manage')
    os.system('chmod +x src/runserver')

    # Create the primary app for containing models/urls/views.
    if not os.path.isdir('src/%s' % project_name):
        os.system('cd src; ./manage startapp %s' % (project_name,))

    os.system('cd src; ./manage syncdb')

class ProjectSatchel(ContainerSatchel):

    name = 'project'

    def set_defaults(self):
        pass

    def update_settings(self, d, role, path='roles/{role}/settings.yaml'):
        """
        Writes a key/value pair to a settings file.
        """
        try:
            import ruamel.yaml
            load_func = ruamel.yaml.round_trip_load
            dump_func = ruamel.yaml.round_trip_dump
        except ImportError:
            print('Warning: ruamel.yaml not available, reverting to yaml package, possible lost of formatting may occur.')
            import yaml
            load_func = yaml.load
            dump_func = yaml.dump
        settings_fn = path.format(role=role)
        data = load_func(open(settings_fn))
        data.update(d)
        settings_str = dump_func(data)
        open(settings_fn, 'w').write(settings_str)

    @task
    def create_skeleton(self, project_name, roles='', components='', pip_requirements='', virtualenv_dir='.env', **kwargs):

        assert project_name, 'Specify project name.'
        site_name = project_name

        app_name = project_name

        default_roles = [_ for _ in roles.split(',') if _.strip()]
        default_components = [_.strip().lower() for _ in components.split(',') if _.strip()]

        print('Creating folders...')
        md('roles/all')
        for _role in default_roles:
            md('roles/%s' % _role)
        md('src')

        print('Creating roles...')
        open('roles/all/settings.yaml', 'w').write(
            self.render_to_string(
                'burlap/all_settings.yaml.template',
                extra=dict(project_name=project_name, site_name=site_name, app_name=app_name)))
        for _role in default_roles:
            open('roles/%s/settings.yaml' % _role, 'w').write(
                self.render_to_string(
                    'burlap/role_settings.yaml.template',
                    extra=dict(project_name=project_name, site_name=site_name, role=_role)))

        default_packages = pip_requirements.split(',')
        if default_packages:
            open('roles/all/pip-requirements.txt', 'w').write('\n'.join(default_packages))

        print('Adding global apt-requirements.txt...')
        open('roles/all/apt-requirements.txt', 'w').write('')

        print('Adding fabfile...')
        content = open(fabfile_template, 'r').read()
        content = content.format(project_name=project_name)
        open('fabfile.py', 'w').write(content.strip()+'\n')

        print('Initializing local development virtual environment...')
        os.system('virtualenv --no-site-packages %s' % virtualenv_dir)
        for package in default_packages:
            os.system('. %s/bin/activate; pip install %s; deactivate' % (virtualenv_dir, package))

        # Install burlap dependencies.
        burlap_pip_requirements = os.path.join(os.path.dirname(burlap.__file__), 'fixtures/requirements.txt')
        print('burlap_pip_requirements:', burlap_pip_requirements)
        assert os.path.exists(burlap_pip_requirements), 'Missing requirements file: %s' % burlap_pip_requirements
        for package in open(burlap_pip_requirements, 'r').readlines():
            if not package.strip():
                continue
            cmd = '%s/bin/pip install %s' % (virtualenv_dir, package)
            print('cmd:', cmd)
            assert not os.system(cmd)

        print('Adding bash setup...')
        open('setup.bash', 'w').write(self.render_to_string('burlap/setup.bash.template'))

        print('Adding gitignore...')
        open('.gitignore', 'w').write(self.render_to_string('burlap/gitignore.template'))

        args = kwargs.copy()
        args['project_name'] = project_name
        args['roles'] = roles
        args['default_roles'] = default_roles
        args['components'] = components
        args['pip_requirements'] = pip_requirements
        args['virtualenv_dir'] = virtualenv_dir
        for component in default_components:
            print('Setting up component %s...' % component)
            # Get component-specific settings.
            component_kwargs = dict(args)
            for _k, _v in kwargs.items():
                _key = component+'_'
                if _k.startswith(_key):
                    component_kwargs[_k[len(_key):]] = _v
                    del component_kwargs[_k]
            print('component_kwargs:', component_kwargs)
            try:
                globals()['init_%s' % component](**component_kwargs)
            except KeyError:
                pass

        print('='*80)
        print()
        print('Skeleton created for project %s!' % (project_name.title(),))
        print()

    @task
    def add_roles(self, roles):
        for role in roles:
            _role = role.strip().lower()
            fn = 'roles/%s/settings.yaml' % _role
            if os.path.isfile(fn):
                continue
            fn_dir = os.path.split(fn)[0]
            if not os.path.isdir(fn_dir):
                os.makedirs(fn_dir)
            open(fn, 'w').write(
                self.render_to_string('burlap/role_settings.yaml.template', extra=dict(role=_role)))
            print('Added role %s!' % role)

    @task
    def create_satchel(self, name):
        name_simple = re.sub(r'[^a-z0-9]+', '', name.lower())
        content = self.render_to_string(
            'burlap/satchel.py.template',
            extra=dict(
                name_camelcase=to_camelcase(name),
                name_simple=name_simple,
            ))
        if not os.path.isdir('satchels'):
            os.makedirs('satchels')
            os.system('touch satchels/__init__.py')
        satchel_fn = 'satchels/%s.py' % name_simple
        open(satchel_fn, 'w').write(content.strip()+'\n')
        print('Wrote %s.' % satchel_fn)

project = ProjectSatchel()

{'ruamel.yaml': 'ruamel.yaml', 'yaml': 'yaml'}


from flask import Blueprint, jsonify

from app.web_app.announcements.models import Announcement, db

mobile = Blueprint('mobile_announcements', __name__, url_prefix='/mobile/announcements')

@mobile.route('/get/<int:announcement_id>')
def get_announcement(announcement_id):
    announcement = Announcement.query.get(announcement_id)

    if announcement:
        announcement.downloads += 1
        db.session.add(announcement)
        db.session.commit()
        return jsonify({
            "error": "None",
            "announcement": {
                "subject": announcement.title,
                "content": announcement.content,
                "ref_number": announcement.ref_number,
                "date_posted": announcement.date_posted
            },
            "sender": {
                "name": announcement.sender.first_name + announcement.sender.last_name,
                "department": announcement.sender.department.name
            }
        })
    else:
        return jsonify({
            "error": "NotFound"
        })


import os
import random
import time
import math
import wikipedia
from google_trans_new import google_translator
import torch
from torch.autograd import Variable
import torch.nn as nn
import numpy as np

CURR_DIR = os.path.dirname(os.path.realpath(__file__))
DATA_DIR = os.path.join(CURR_DIR, '../data')
EMBEDDING_DIM = 300
EMBEDDINGS_PATH = os.path.join(CURR_DIR, "embeddings.txt")
HIDDEN_SIZE = 2048
NUM_LAYERS = 2
CHUNK_LEN = 400
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
BATCH_SIZE = 128
TEST_DATA = test_data = [
    ('Happ', 'y'),
    ('Happy Ne', 'w'),
    ('Happy New Yea', 'r'),
    ('That’s one small ste', 'p'),
    ('That’s one sm', 'a'),
    ('That’', 's'),
    ('Th', 'a'),
    ('one giant leap for mankin', 'd'),
    ('one giant leap fo', 'r'),
    ('one giant lea', 'p'),
    ('one giant l', 'e'),
    ('one gia', 'n'),
    ('on', 'e'),
    ('Ruin h', 'a'),
    ('Ruin has co', 'm'),
    ('Ruin has come to o', 'u'),
    ('Ruin has come to our fam', 'i'),
    ('Ruin has come to our famil', 'y'),
    ('You rem', 'e'),
    ('You remember our vene', 'r'),
    ('You remember our venerabl', 'e'),
    ('You remember our venerable hou', 's'),
    ('You remember our venerable house, op', 'u'),
    ('You remember our venerable house, opulent and im', 'p'),
    ('You remember our venerable house, opulent and imperia', 'l'),
    ('G', 'a'),
    ('Gaz', 'i'),
    ('Gazin', 'g'),
    ('Gazing pro', 'u'),
    ('Gazing proudl', 'y'),
    ('Gazing proudly from its sto', 'i'),
    ('Gazing proudly from its stoic per', 'c'),
    ('Gazing proudly from its stoic perch ab', 'o'),
    ('Gazing proudly from its stoic perch abo', 'v'),
    ('Gazing proudly from its stoic perch above the moo', 'r'),
    ('Mons', 't'),
    ('Monst', 'r'),
    ('Monstrou', 's'),
    ('Monstrous si', 'z'),
    ('Monstrous size has no in', 't'),
    ('Monstrous size has no intrins', 'i'),
    ('Monstrous size has no intrinsic mer', 'i'),
    ('Monstrous size has no intrinsic meri', 't'),
    ('Monstrous size has no intrinsic merit, unl', 'e'),
    ('Monstrous size has no intrinsic merit, unless inor', 'd'),
    ('Monstrous size has no intrinsic merit, unless inordinat', 'e'),
    ('Monstrous size has no intrinsic merit, unless inordinate e', 'x'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsan', 'g'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsanguinatio', 'n'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsanguination be co', 'n'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsanguination be con', 's'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsanguination be consid', 'e'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsanguination be conside', 'r'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsanguination be consider', 'e'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsanguination be considere', 'd'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsanguination be considered a vir', 't'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsanguination be considered a virt', 'u'),
    ('Monstrous size has no intrinsic merit, unless inordinate exsanguination be considered a virtu', 'e'),
]
random.seed(447)

def get_vocab():
  vocab = set()
  file_list = os.listdir(DATA_DIR)
  for i, file in enumerate(file_list):
    with open(os.path.join(DATA_DIR, file)) as f:
      text = f.read()
      f_vocab = set(text)
      currlen = len(vocab)
      vocab.update(f_vocab)
      # print("{}/{} Added {} characters to vocab from {}".format(i+1, len(file_list), len(vocab) - currlen, file))
  voc_list = list(vocab)
  voc_list.insert(0, '\u2400')
  voc_string = ''.join(voc_list)

  print("{} total characters in vocab".format(len(vocab)))
  return voc_string

with open(os.path.join(CURR_DIR, 'vocab.txt'), 'r') as f:
  vocab = f.read()
NUM_CHARACTERS = len(vocab)

def time_since(since):
  s = time.time() - since
  m = math.floor(s / 60)
  s -= m * 60
  return '%dm %ds' % (m, s)

def char_tensor(s: str):
  if torch.cuda.is_available():
    tensor = torch.zeros(len(s)).to(torch.int64).cuda('cuda')
  else:
    tensor = torch.zeros(len(s)).to(torch.int64)
  for c in range(len(s)):
    try:
      tensor[c] = vocab.index(s[c])
    except ValueError:
      tensor[c] = 0 # if not in vocab, use 0 as index. 0 indexes the null symbol in vocab
  return Variable(tensor)

class TrainData():
  def __init__(self, batch_size=BATCH_SIZE):
    self.batch_index = 0
    self.batch_size = batch_size
    data = []
    file_list = os.listdir(DATA_DIR)
    non_cuda_tensors = 0
    print(len(file_list), 'articles in the dataset')
    hundreth = len(file_list) // 100
    tenth = len(file_list) // 10
    for i, filename in enumerate(file_list):
      if (i+1) % tenth == 0:
        print('|', end='')
      elif (i+1) % hundreth == 0:
        print('-', end='')
      with open(os.path.join(DATA_DIR, filename), 'r') as f:
        for line in f:
          line = line.strip('\n')
          if len(line) <= 1:
            continue
          elif len(line) > CHUNK_LEN:
            idx = 0
            while idx < len(line):
              if idx + CHUNK_LEN < len(line):
                chunk = line[idx:idx+CHUNK_LEN]
              else:
                chunk = line[idx:]
              tensor = char_tensor(chunk)
              if not tensor.is_cuda:
                non_cuda_tensors += 1
              data.append(tensor)
              idx += CHUNK_LEN
          else:
            tensor = char_tensor(line)
            if not tensor.is_cuda:
              non_cuda_tensors += 1
            data.append(tensor)
    random.shuffle(data)
    print(f'\nNon-cuda tensors in dataset: {non_cuda_tensors}')
    print(f'Total tensors in dataset: {len(data)}')
    self.data = data
  def random_training_set(self):
    if self.batch_index + self.batch_size >= len(self.data):
      self.batch_index = 0
      return self.data[self.batch_index:]
    else:
      self.batch_index += self.batch_size
      return self.data[self.batch_index:self.batch_index + self.batch_size]

def get_tensor_embeddings(input_size, embedding_dim, vocab):
  embeddings = {}
  try:
    file = open(EMBEDDINGS_PATH, 'r')
    for line in file:
      raw = line.strip().split()
      #First value in the line is the character name the rest are float values
      embedValues = np.asarray(raw[1:], dtype=float)
      char = raw[0]
      embeddings[char] = embedValues
  except:
    pass

  if torch.cuda.is_available():
    tensor_embeddings = torch.normal(0,1,(input_size, embedding_dim)).cuda('cuda')
  else:
    tensor_embeddings = torch.normal(0,1,(input_size, embedding_dim))
  for i, char in enumerate(vocab):
    if char in embeddings:
      tensor_embeddings[i] = torch.Tensor(embeddings[char])
  return tensor_embeddings

class Model(nn.Module):
  def __init__(self, vocab=vocab, input_size=NUM_CHARACTERS, hidden_size=HIDDEN_SIZE, output_size=NUM_CHARACTERS, n_layers=NUM_LAYERS):
    super(Model, self).__init__()
    self.vocab = vocab
    self.input_size = input_size
    self.hidden_size = hidden_size
    self.output_size = output_size
    self.n_layers = n_layers
    tensor_embeddings = get_tensor_embeddings(input_size, EMBEDDING_DIM, self.vocab)
    self.encoder = nn.Embedding.from_pretrained(tensor_embeddings, freeze=False)
    self.gru = nn.GRU(EMBEDDING_DIM, hidden_size, n_layers)
    self.decoder = nn.Linear(hidden_size, output_size)

  def forward(self, inputchar, hidden):
    inputchar = self.encoder(inputchar.view(1, -1))
    output, hidden = self.gru(inputchar.view(1, 1, -1), hidden)
    output = self.decoder(output.view(1, -1))
    return output, hidden

  def init_hidden(self):
    if torch.cuda.is_available():
      return Variable(torch.zeros(self.n_layers, 1, self.hidden_size).cuda('cuda'))
    else:
      return Variable(torch.zeros(self.n_layers, 1, self.hidden_size))

  def train_step(self, inp, criterion, optim,):
    self.zero_grad()
    loss = 0
    total_items = 0
    for i, line in enumerate(inp):
      if (i+1) % (BATCH_SIZE // 10) == 0:
        print('|', end='')
      elif (i+1) % (BATCH_SIZE // 100) == 0:
        print('-', end='')
      hidden = self.init_hidden()
      total_items += len(line) - 1
      for c in range(len(line) - 1):
        output, hidden = self(line[c], hidden)
        target = line[c + 1]
        # unsqueeze() is used to add dimension to the tensor
        loss += criterion(output, target.unsqueeze(dim=0))
    # Back propagation
    print("\nUpdating weights")
    loss.backward()
    optim.step()
    return loss.item() / total_items

  def predict(self, history='A'):
    self.eval()
    hidden = self.init_hidden()
    history_input = char_tensor(history)

    # Use priming string to "build up" hidden state
    for c in range(len(history) - 1):
      _, hidden = self(history_input[c], hidden)
    inp = history_input[-1]

    output, hidden = self(inp, hidden)
    # print(output, type(output))
    top_i = []
    for i in torch.argsort(output[0])[-3:]:
      top_i.append(i.item())

    # Add predicted character to string and use as next input
    predicted_chars = []
    for i in top_i:
      predicted_chars.append(self.vocab[i])
    self.train()
    return predicted_chars

  # test data is a tuple of strings. first string is history, next string is correct char
  def evaluate(self, test_data=test_data):
    total = len(test_data)
    correct = 0
    for history, next_char in test_data:
      preds = self.predict(history)
      if next_char in preds:
        correct += 1
    return correct / total

def run_train(model, train_data, work_dir):
  model.to(DEVICE)
  for param in model.parameters():
    if not param.is_cuda:
      print("Model not initialized as cuda")
      break

  optim = torch.optim.Adam(model.parameters(), lr=0.0005)
  criterion = nn.CrossEntropyLoss()
  start = time.time()

  eps = 250
  with open(os.path.join(work_dir, 'train_log.txt'), 'w') as f:
    print("Training model")
    for epoch in range(1, eps + 1):
      print(f"Epoch {epoch}")
      inp = train_data.random_training_set()
      loss = model.train_step(inp, criterion, optim)
      accuracy = (model.evaluate() * 100)
      epoch_summary = '[%s (%d %d%%) %.4f], Accuracy: %.3f%%' % (time_since(start), epoch, epoch / eps * 100, loss, accuracy)
      print(epoch_summary)
      f.write(epoch_summary)
      f.write('\n')

  with open('model.checkpoint.pt', 'wb') as f:
    torch.save(model, f)

  for history, next_char in test_data:
    preds = model.predict(history)
    print(next_char, preds, history)


import torch
import numpy as np
class MyDiscriminator(torch.nn.Module):
    def __init__(self, img_shape):
        super(MyDiscriminator, self).__init__()
        self.hidden_dim = 128
        #int(np.prod(img_shape))
        self.model = torch.nn.Sequential(
            torch.nn.Conv2d(3, self.hidden_dim, 5, stride=2, padding=2),
            torch.nn.LeakyReLU(inplace=True),
            torch.nn.Conv2d(self.hidden_dim, self.hidden_dim * 2, 5, stride=2, padding=2),
            torch.nn.LeakyReLU(True),
            torch.nn.BatchNorm2d(self.hidden_dim * 2, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),
            torch.nn.Conv2d(2 * self.hidden_dim, 4 * self.hidden_dim, 5, stride=2, padding=2),
            torch.nn.LeakyReLU(True),
        )
        self.linear = torch.nn.Linear(4 * self.hidden_dim * 8 * 8, 1)

    def forward(self, img):
        #print("dis")
        if(img.shape[1] != 3):
            img = img.transpose(3,2).transpose(2,1)
        out = self.model(img)
        #print(out.shape)
        out = out.view(img.shape[0], -1)
        #print(img.shape[0])
        #print(out.shape)
        out = self.linear(out)
        #print(out.shape)
        return out.squeeze(-1)


from flask import Flask, render_template , request
import mlab
from models.superbike import Bike

mlab.connect()

app = Flask(__name__)

@app.route("/add_bike", methods = ["GET","POST"])
def add_bike():
    if request.method == "GET":
        return render_template("add_bike.html")
    if request.method == "POST":
        form = request.form
        bikename = form["bikename"]
        hangxe = form["hangxe"]
        image = form["image"]
        dongco = form["dongco"]
        maluc = form["maluc"]
        hopso = form["hopso"]
        chieucaoyenxe = form["chieucaoyenxe"]
        dacdiemkhac = form["dacdiemkhac"]
        giaxe = form["giaxe"]
        superbike = Bike(bikename = bikename,hangxe = hangxe, image = image ,dongco = dongco , maluc = maluc , hopso = hopso , chieucaoyenxe = chieucaoyenxe  , dacdiemkhac = dacdiemkhac , giaxe = giaxe)
        superbike.save()
        return render_template("add_bike.html")

if __name__ == "__main__":
    app.run(debug=True)

# bike = Bike.objects.with_id("5c1bc34abd42b10f5051e379")

# bike.update(set__bikename = " CB 500F")
# print(bike.bikename)


import numpy as np
import time
import os
import torch.nn as nn
import torch
from torch import optim
from torch.utils.data import Dataset, DataLoader
from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence
from Data_utils import VocabDataset, vocab_collate_func
from preprocessing_util import preposs_toekn, Lang, text2index, construct_Lang
from selfAtten_RNN_Encoder import Encoder
from Multilayers_Decoder import DecoderAtten
from selfatten_RNN_evaluation import evaluate_batch
from config import device, PAD_token, SOS_token, EOS_token, UNK_token, embedding_freeze, vocab_prefix
import random
import pickle
from SelfAtten_RNN_config import args
from torch.optim.lr_scheduler import LambdaLR

####################Define Global Variable#########################

#device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
#print(device)

####################Define Global Variable#########################

def train(input_tensor, input_lengths, target_tensor, target_lengths,
          encoder, decoder, encoder_optimizer, decoder_optimizer, criterion,
          teacher_forcing_ratio):
    '''
    finish train for a batch
    '''
    batch_size = input_tensor.size(0)
    encoder_optimizer.zero_grad()
    decoder_optimizer.zero_grad()

    loss = 0
    encoder_outputs, encoder_hidden, encoder_cell = encoder(input_tensor, input_lengths)

    decoder_input = torch.tensor([[SOS_token]*batch_size], device=device).transpose(0,1)
    decoder_hidden,decoder_cell = encoder_hidden, encoder_cell  #decoder.initHidden(encoder_hidden)
    #print(decoder_hidden.size())
    #print('encoddddddddddder finishhhhhhhhhhhhhhh')
    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False

    if use_teacher_forcing:
        #### Teacher forcing: Feed the target as the next input
        # target_lengths = target_lengths.cpu().numpy()
        # sent_not_end_index = list(range(batch_size))
        # decoding_token_index = 0
        # while len(sent_not_end_index) > 0:
        #     decoder_output, decoder_hidden, decoder_attention = decoder(
        #         decoder_input, decoder_hidden, input_lengths, encoder_outputs)
        #     sent_not_end_index = torch.LongTensor(sent_not_end_index).to(device)
        #     loss += criterion(decoder_output.index_select(0,sent_not_end_index),
        #                       target_tensor[:,decoding_token_index].index_select(0,sent_not_end_index))
        #     decoder_input = target_tensor[:,decoding_token_index].unsqueeze(1)  # Teacher forcing
        #     decoding_token_index += 1
        #     end_or_not = target_lengths > decoding_token_index
        #     sent_not_end_index = list(np.where(end_or_not)[0])
        ### simple version;
        decoding_token_index = 0
        tgt_max_len_batch = target_lengths.cpu().max().item()
        assert(tgt_max_len_batch==target_tensor.size(1))
        while decoding_token_index < tgt_max_len_batch:
            decoder_output, decoder_hidden, decoder_attention, decoder_cell = decoder(
                decoder_input, decoder_hidden, input_lengths, encoder_outputs, decoder_cell)
            loss += criterion(decoder_output, target_tensor[:,decoding_token_index])
            decoder_input = target_tensor[:,decoding_token_index].unsqueeze(1)  # Teacher forcing
            decoding_token_index += 1

    else:
        ### debug
        # # Without teacher forcing: use its own predictions as the next input
        # target_lengths_numpy = target_lengths.cpu().numpy()
        # sent_not_end_index = list(range(batch_size))
        # decoding_token_index = 0
        # while len(sent_not_end_index) > 0:
        #     decoder_output, decoder_hidden, decoder_attention_weights = decoder(
        #         decoder_input, decoder_hidden, input_lengths, encoder_outputs)
        #     topv, topi = decoder_output.topk(1)
        #     decoder_input = topi.detach()  # detach from history as input
        #     #print(type(sent_not_end_index[0]))
        #     sent_not_end_index = torch.LongTensor(sent_not_end_index).to(device)
        #     loss += criterion(decoder_output.index_select(0,sent_not_end_index),
        #                       target_tensor[:,decoding_token_index].index_select(0,sent_not_end_index))
        #     decoding_token_index += 1
        #     end_or_not = target_lengths_numpy > decoding_token_index
        #     #(target_lengths_numpy > decoding_token_index)*(decoder_input.squeeze().numpy() != EOS_token)
        #     sent_not_end_index = list(np.where(end_or_not)[0])
        ### simple version
        decoding_token_index = 0
        tgt_max_len_batch = target_lengths.cpu().max().item()
        assert(tgt_max_len_batch==target_tensor.size(1))
        while decoding_token_index < tgt_max_len_batch:
            decoder_output, decoder_hidden, decoder_attention_weights, decoder_cell = decoder(
                decoder_input, decoder_hidden, input_lengths, encoder_outputs, decoder_cell)
            topv, topi = decoder_output.topk(1)
            decoder_input = topi.detach()  # detach from history as input
            loss += criterion(decoder_output, target_tensor[:,decoding_token_index])
            decoding_token_index += 1

    # average loss
    #target_lengths.type_as(loss).mean()
    loss.backward()

    ### TODO
    # clip for gradient exploding
    encoder_optimizer.step()
    decoder_optimizer.step()

    return loss.item()/tgt_max_len_batch  #torch.div(loss, target_lengths.type_as(loss).mean()).item()  #/target_lengths.mean()

def trainIters(train_loader, val_loader, encoder, decoder, num_epochs,
               learning_rate, teacher_forcing_ratio, srcLang, tgtLang, model_save_info, tgt_max_len, beam_size):

    encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate, betas=(0.9, 0.98), eps=1e-09)
    decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate, betas=(0.9, 0.98), eps=1e-09)

    if model_save_info['model_path_for_resume'] is not None:
        check_point_state = torch.load(model_save_info['model_path_for_resume'])
        encoder.load_state_dict(check_point_state['encoder_state_dict'])
        encoder_optimizer.load_state_dict(check_point_state['encoder_optimizer_state_dict'])
        decoder.load_state_dict(check_point_state['decoder_state_dict'])
        decoder_optimizer.load_state_dict(check_point_state['decoder_optimizer_state_dict'])

    lr_decay = False
    def lr_foo(step,warmup_steps = 4000):
        if step < 500:
            return 0.001/(step+1)
        else:
            return 1.0/np.sqrt(args['encoder_embed_dim'])* min(1.0/np.sqrt(step), step*warmup_steps**(-1.5))

    criterion = nn.NLLLoss() #nn.NLLLoss(ignore_index=PAD_token)
    max_val_bleu = 0

    if lr_decay:
        lambda_T = lambda step: lr_foo(step)
        scheduler_encoder = LambdaLR(encoder_optimizer, lr_lambda=lambda_T)
        scheduler_decoder = LambdaLR(decoder_optimizer, lr_lambda=lambda_T)

    for epoch in range(num_epochs):
        n_iter = -1
        start_time = time.time()
        for input_tensor, input_lengths, target_tensor, target_lengths in train_loader:
            n_iter += 1

            if lr_decay:
                scheduler_encoder.step()
                scheduler_decoder.step()
            #print('start_step: ', n_iter)
            loss = train(input_tensor, input_lengths, target_tensor, target_lengths,
                         encoder, decoder, encoder_optimizer, decoder_optimizer,
                         criterion, teacher_forcing_ratio)
            if n_iter % 200 == 0:
                print('Loss:', loss)
                #eva_start = time.time()
                #val_bleu_sacre, val_bleu_nltk, val_loss = evaluate_batch(val_loader, encoder, decoder, criterion, tgt_max_len, tgtLang.index2word, srcLang.index2word)
                #print((time.time()-eva_start)/60)
                #print('epoch: [{}/{}], step: [{}/{}], train_loss:{}, val_bleu_sacre: {}, val_bleu_nltk: {}, val_loss: {}'.format(
               #     epoch, num_epochs, n_iter, len(train_loader), loss, val_bleu_sacre[0], val_bleu_nltk, val_loss))
               # print('Decoder parameters grad:')
               # for p in decoder.named_parameters():
               #     print(p[0], ': ',  p[1].grad.data.abs().mean().item(), p[1].grad.data.abs().max().item(), p[1].data.abs().mean().item(), p[1].data.abs().max().item(), end=' ')
               # print('\n')
               # print('Encoder Parameters grad:')
               # for p in encoder.named_parameters():
               #     print(p[0], ': ',  p[1].grad.data.abs().mean().item(), p[1].grad.data.abs().max().item(), p[1].data.abs().mean().item(), p[1].data.abs().max().item(), end=' ')
               # print('\n')
        val_bleu_sacre, val_bleu_nltk, val_loss = evaluate_batch(val_loader, encoder, decoder, criterion, tgt_max_len, tgtLang.index2word, srcLang.index2word)
        print('epoch: [{}/{}] (Running time {:.3f} min), val_bleu_sacre: {}, val_bleu_nltk: {}, val_loss: {}'.format(epoch, num_epochs, (time.time()-start_time)/60, val_bleu_sacre, val_bleu_nltk, val_loss))
        #val_bleu_sacre_beam, _, _ = evaluate_beam_batch(beam_size, val_loader, encoder, decoder, criterion, tgt_max_len, tgtLang.index2word)
        #print('epoch: [{}/{}] (Running time {:.3f} min), val_bleu_sacre_beam: {}'.format(epoch, num_epochs, (time.time()-start_time)/60, val_bleu_sacre_beam))
        if max_val_bleu < val_bleu_sacre.score:
            max_val_bleu = val_bleu_sacre.score
            ### TODO save best model
        if (epoch+1) % model_save_info['epochs_per_save_model'] == 0:
            check_point_state = {
                'epoch': epoch,
                'encoder_state_dict': encoder.state_dict(),
                'encoder_optimizer_state_dict': encoder_optimizer.state_dict(),
                'decoder_state_dict': decoder.state_dict(),
                'decoder_optimizer_state_dict': decoder_optimizer.state_dict()
                }
            torch.save(check_point_state, '{}epoch_{}.pth'.format(model_save_info['model_path'], epoch))

    return None

def start_train(transtype, paras):
    src_max_vocab_size = paras['src_max_vocab_size']
    tgt_max_vocab_size = paras['tgt_max_vocab_size']
    tgt_max_len = paras['tgt_max_len']
    max_src_len_dataloader = paras['max_src_len_dataloader']
    max_tgt_len_dataloader = paras['max_tgt_len_dataloader']

    teacher_forcing_ratio = paras['teacher_forcing_ratio']
    emb_size = paras['emb_size']
    hidden_size = paras['hidden_size']
    num_layers = paras['num_layers']
    num_direction = paras['num_direction']
    deal_bi = paras['deal_bi']
    learning_rate = paras['learning_rate']
    num_epochs = paras['num_epochs']
    batch_size = paras['batch_size']
    rnn_type = paras['rnn_type']
    attention_type = paras['attention_type']
    beam_size = paras['beam_size']
    model_save_info = paras['model_save_info']
    dropout_rate = paras['dropout_rate']

    address_book=dict(
        train_src = 'Machine_Translation_NLP/iwsltzhen/iwslt-{}-{}/train.tok.{}'.format(transtype[0], transtype[1], transtype[0]),
        train_tgt = 'Machine_Translation_NLP/iwsltzhen/iwslt-{}-{}/train.tok.{}'.format(transtype[0], transtype[1], transtype[1]),
        val_src = 'Machine_Translation_NLP/iwsltzhen/iwslt-{}-{}/dev.tok.{}'.format(transtype[0], transtype[1], transtype[0]),
        val_tgt = 'Machine_Translation_NLP/iwsltzhen/iwslt-{}-{}/dev.tok.{}'.format(transtype[0], transtype[1], transtype[1]),
        src_emb = 'embedding/wiki.{}.vec'.format(transtype[0]),
        tgt_emb = 'embedding/wiki.{}.vec'.format(transtype[1])
        )
    #print(address_book)
    train_src_add = address_book['train_src']
    train_tgt_add = address_book['train_tgt']
    val_src_add = address_book['val_src']
    val_tgt_add = address_book['val_tgt']
    # make dir for saving models
    if not os.path.exists(model_save_info['model_path']):
        os.makedirs(model_save_info['model_path'])
    ### save model hyperparameters
    with open(model_save_info['model_path']+'model_params.pkl', 'wb') as f:
        model_hyparams = paras
        model_hyparams['address_book'] = address_book
        pickle.dump(model_hyparams, f)
    print(model_hyparams)

    train_src = []
    with open(train_src_add) as f:
        for line in f:
            train_src.append(preposs_toekn(line[:-1].strip().split(' ')))

    train_tgt = []
    with open(train_tgt_add) as f:
        for line in f:
            train_tgt.append(preposs_toekn(line[:-1].strip().split(' ')))

    val_src = []
    with open(val_src_add) as f:
        for line in f:
            val_src.append(preposs_toekn(line[:-1].strip().split(' ')))

    val_tgt = []
    with open(val_tgt_add) as f:
        for line in f:
            val_tgt.append(preposs_toekn(line[:-1].strip().split(' ')))

    print('The number of train samples: ', len(train_src))
    print('The number of val samples: ', len(val_src))
    srcLang = construct_Lang('src', src_max_vocab_size, address_book['src_emb'], train_src)
    tgtLang = construct_Lang('tgt', tgt_max_vocab_size, address_book['tgt_emb'], train_tgt)
    train_input_index = text2index(train_src, srcLang.word2index) #add EOS token here
    train_output_index = text2index(train_tgt, tgtLang.word2index)
    val_input_index = text2index(val_src, srcLang.word2index)
    val_output_index = text2index(val_tgt, tgtLang.word2index)
    ### save srcLang and tgtLang

    train_dataset = VocabDataset(train_input_index,train_output_index, max_src_len_dataloader, max_tgt_len_dataloader)
    train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                               batch_size=batch_size,
                                               collate_fn=vocab_collate_func,
                                               shuffle=True)

    val_dataset = VocabDataset(val_input_index,val_output_index, None, None)
    val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
                                            batch_size=batch_size,
                                            collate_fn=vocab_collate_func,
                                            shuffle=False)

    # test_dataset = VocabDataset(test_data)
    # test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
    #                                            batch_size=BATCH_SIZE,
    #                                            collate_fn=vocab_collate_func,
    #                                            shuffle=False)

    embedding_src_weight = torch.from_numpy(srcLang.embedding_matrix).type(torch.FloatTensor).to(device)
    embedding_tgt_weight = torch.from_numpy(tgtLang.embedding_matrix).type(torch.FloatTensor).to(device)
    print(embedding_src_weight.size(), embedding_tgt_weight.size())
    if attention_type:
        encoder = Encoder(args, num_layers, vocab_size = srcLang.vocab_size, use_position_emb = False)
        decoder = DecoderAtten(emb_size, hidden_size, tgtLang.vocab_size, num_layers, rnn_type = rnn_type, atten_type = attention_type, dropout_rate = dropout_rate)
    else:
        encoder = Encoder(args, num_layers, vocab_size = srcLang.vocab_size, use_position_emb = False)
        decoder = DecoderRNN(emb_size, hidden_size, tgtLang.vocab_size, num_layers, rnn_type = rnn_type, dropout_rate = dropout_rate)

    encoder, decoder = encoder.to(device), decoder.to(device)
    print('Encoder:')
    print(encoder)
    print('Decoder:')
    print(decoder)
    trainIters(train_loader, val_loader, encoder, decoder, num_epochs, learning_rate, teacher_forcing_ratio, srcLang, tgtLang, model_save_info, tgt_max_len, beam_size)

if __name__ == "__main__":
    transtype = ('vi', 'en')
    paras = dict(
        src_max_vocab_size = 26109, # 47127, #26109,
        tgt_max_vocab_size = 24418, #31553, #24418,
        tgt_max_len = 128,
        max_src_len_dataloader = 72, #67, #72,
        max_tgt_len_dataloader = 71, #72, #71,

        emb_size = 300,
        hidden_size = 300,
        num_layers = 2,
        num_direction = 1,
        deal_bi = 'linear', #{'linear', 'sum'}
        rnn_type = 'LSTM', # LSTM
        attention_type = 'concat', #'dot_prod', general, concat
        teacher_forcing_ratio = 1,

        learning_rate = 3e-4,
        num_epochs = 50,
        batch_size = 128,
        beam_size = 1,
        dropout_rate = 0.1,

        model_save_info = dict(
            model_path = 'nmt_models/vi-en-selfattention300-lrdecay_encoderlayer21_random_dr1_lige/',
            epochs_per_save_model = 2,
            model_path_for_resume = None #'nmt_models/epoch_0.pth'
            )
        )
    #print('paras: ', paras)
    start_train(transtype, paras)


import mock
import pytest

from bson.objectid import ObjectId
from nbsearch.v1 import query

from .utils import AsyncMock, AsyncIterator

@pytest.mark.asyncio
async def test_mongo_query_from_q():
    nq = query.nq_from_q('QUERY')
    assert nq == {'target': {'text': 'QUERY', 'type': 'all'}}

    history = mock.Mock()
    notebooks = mock.Mock()
    mongoq = await query.mongo_agg_query_from_nq(nq, history, notebooks)
    assert mongoq == [{'$match': {'$text': {'$search': 'QUERY'}}}]

@pytest.mark.asyncio
async def test_mongo_query_from_meme():
    nq = query.nq_from_meme('MEME')
    assert nq == {'cell': {'and': [{'in_meme': 'MEME'}]}}

    history = mock.Mock()
    notebooks = mock.Mock()
    mongoq = await query.mongo_agg_query_from_nq(nq, history, notebooks)
    assert mongoq == [{'$match': {'$and': [{'cells': {'$elemMatch': {'metadata.lc_cell_meme.current': {'$regex': '.*MEME.*'}}}}]}}]

@pytest.mark.asyncio
async def test_mongo_query_from_invalid_nq():
    history = mock.Mock()
    notebooks = mock.Mock()

    with pytest.raises(KeyError):
        nq = {'cell': {'and': [{'not_valid': 'OUTPUT'}]}}
        await query.mongo_agg_query_from_nq(nq, history, notebooks)

@pytest.mark.asyncio
async def test_mongo_cell_query_from_nq():
    history = mock.Mock()
    notebooks = mock.Mock()

    nq = {'cell': {'and': [{'in_code': 'CODE'}]}}
    mongoq = await query.mongo_agg_query_from_nq(nq, history, notebooks)
    assert mongoq == [{'$match': {'$and': [{'cells': {'$elemMatch': {'cell_type': 'code', 'source': {'$regex': '.*CODE.*'}}}}]}}]

    nq = {'cell': {'and': [{'not_in_code': 'CODE'}]}}
    mongoq = await query.mongo_agg_query_from_nq(nq, history, notebooks)
    assert mongoq == [{'$match': {'$and': [{'cells': {'$elemMatch': {'cell_type': 'code', 'source': {'$not': {'$regex': '.*CODE.*'}}}}}]}}]

    nq = {'cell': {'and': [{'not_in_code': 'CODE', 'in_output': 'OUTPUT'}]}}
    mongoq = await query.mongo_agg_query_from_nq(nq, history, notebooks)
    assert mongoq == [{'$match': {'$and': [{'cells': {'$elemMatch': {'cell_type': 'code', 'source': {'$not': {'$regex': '.*CODE.*'}}, 'cell.outputs.text': {'$regex': '.*OUTPUT.*'}}}}]}}]

    nq = {'cell': {'and': [{'in_markdown': 'MARKDOWN'}]}}
    mongoq = await query.mongo_agg_query_from_nq(nq, history, notebooks)
    assert mongoq == [{'$match': {'$and': [{'cells': {'$elemMatch': {'cell_type': 'markdown', 'source': {'$regex': '.*MARKDOWN.*'}}}}]}}]

    nq = {'cell': {'and': [{'in_output': 'OUTPUT'}]}}
    mongoq = await query.mongo_agg_query_from_nq(nq, history, notebooks)
    assert mongoq == [{'$match': {'$and': [{'cells': {'$elemMatch': {'cell.outputs.text': {'$regex': '.*OUTPUT.*'}}}}]}}]

@pytest.mark.asyncio
async def test_mongo_target_query_from_nq():
    history = mock.Mock()
    notebooks = mock.Mock()

    nq = {'target': {'text': 'TEXT', 'type': 'all'}}
    mongoq = await query.mongo_agg_query_from_nq(nq, history, notebooks)
    assert mongoq == [{'$match': {'$text': {'$search': 'TEXT'}}}]

    history.find_one = AsyncMock()
    history.find_one.return_value = {'notebook_ids': ['0123456789abcdef01234567', '123456789abcdef012345670']}

    nq = {'target': {'history_in': 'abcdef012345670123456789'}}
    mongoq = await query.mongo_agg_query_from_nq(nq, history, notebooks)
    assert mongoq == [{'$match': {'_id': {'$in': [ObjectId('0123456789abcdef01234567'), ObjectId('123456789abcdef012345670')]}}}]

    notebooks.aggregate = mock.Mock()
    notebooks.aggregate.return_value = AsyncIterator([{'_id': 'MEME{0:04d}'.format(i)} for i in range(5)])

    nq = {'target': {'history_related': 'abcdef012345670123456789'}}
    mongoq = await query.mongo_agg_query_from_nq(nq, history, notebooks)
    print(mongoq)
    assert mongoq == [{'$match': {'cells': {'$elemMatch': {'$or': [
        {'metadata.lc_cell_meme.current': {'$regex': '^MEME0000.*'}},
        {'metadata.lc_cell_meme.current': {'$regex': '^MEME0001.*'}},
        {'metadata.lc_cell_meme.current': {'$regex': '^MEME0002.*'}},
        {'metadata.lc_cell_meme.current': {'$regex': '^MEME0003.*'}},
        {'metadata.lc_cell_meme.current': {'$regex': '^MEME0004.*'}}
    ]}}}}]


import sys
import re
import pathlib
import os
import fileinput

def _get_project_name():
    manage_file_path = os.path.join(pathlib.Path(__file__).parent.absolute(), 'manage.py')
    with open(manage_file_path) as file:
        for line in file.readlines():
            if 'DJANGO_SETTINGS_MODULE' in line:
                return line.split("', '")[-1].split('.settings')[0]

def _search_and_replace(file, occurence, replace_by):
    with fileinput.FileInput(file, inplace=True) as file:
        for line in file:
            print(line.replace(occurence, replace_by), end='')

def rename_project(project_name: str):
    project_name = project_name.lower();
    if (not re.match(r"^([a-z0-9_]{3,20}[^-])$", project_name)):
        sys.exit('Could not change the name of the project.\nThe project name must be lower cased and range from 3 to 20 characters!')

    old_project_name = _get_project_name()
    directory_path = pathlib.Path(__file__).parent.absolute()

    to_search = (
        (old_project_name, 'asgi.py'),
        (old_project_name, 'settings.py'),
        (old_project_name, 'wsgi.py'),
        (old_project_name, 'urls.py'),
        ('manage.py', ),
    )

    for path_sequence in to_search:
        _search_and_replace(os.path.join(directory_path, *path_sequence), old_project_name, project_name)

    os.rename(os.path.join(directory_path, old_project_name), os.path.join(directory_path, project_name))

    if (_get_project_name() == project_name):
        print(f"The Django Project has been renamed to '{project_name}' successfully!")
    else:
        print(f"Something went wrong while renaming the project... The action could not be done!")

if __name__ == "__main__":
    args = sys.argv[1:]
    if len(args) == 0:
        project_name = input('Choose a name for the project:\n     The name should be lower cased alphanumerical value (underscore include) \n     with length ranging from 3 to 20.\n:')
    else:
        project_name = args[0]
    rename_project(project_name)


import setuptools

# A bit of pbr monkey-patching
# We'd like to have a way to compose extras.
import pbr.util
orig_setup_cfg_to_setup_kwargs = pbr.util.setup_cfg_to_setup_kwargs

def lookup_extras(requirement, extras):
    if not requirement.startswith('['):
        return [requirement]
    extra = requirement.strip('[]')
    requirements = []
    for requirement in extras[extra]:
        requirements += lookup_extras(requirement, extras)
    return requirements

def setup_cfg_to_setup_kwargs(*args, **kwargs):
    kwargs = orig_setup_cfg_to_setup_kwargs(*args, **kwargs)
    extras = kwargs['extras_require']
    for extra, requirements in extras.items():
        new_requirements = []
        for requirement in requirements:
            new_requirements += lookup_extras(requirement, extras)
        extras[extra] = new_requirements
    return kwargs

pbr.util.setup_cfg_to_setup_kwargs = setup_cfg_to_setup_kwargs

setuptools.setup(
    setup_requires=['pbr'],
    pbr=True,
)


import gi

gi.require_version("GstBase", "1.0")

import os
import cv2
import numpy as np
import msgpack
import struct
from gi.repository import Gst, GObject, GstBase, GLib
from typing import List

Gst.init(None)

ICAPS = Gst.Caps(
    Gst.Structure(
        "application/msgpack-predicts"
    )
)

OCAPS = Gst.Caps(
    Gst.Structure(
        "application/meter",
    )
)

def read_mask():
    mask = cv2.imread(os.path.join(
        os.path.dirname(os.path.realpath(__file__)),
        "..",
        "resource",
        "mask.bmp",
    ))
    return mask

class PeopleCounter(GstBase.BaseTransform):
    __gstmetadata__ = ("PeopleCounter", "Transform", "Counter", "UM")

    __gsttemplates__ = (
        Gst.PadTemplate.new(
            "src", Gst.PadDirection.SRC, Gst.PadPresence.ALWAYS, OCAPS
        ),
        Gst.PadTemplate.new(
            "sink", Gst.PadDirection.SINK, Gst.PadPresence.ALWAYS, ICAPS
        ),
    )

    max_age = GObject.Property(
        type=int,
        nick="Max Age",
        blurb="Amount of frames, infuencing on current count.",
        minimum=0,
        maximum=100,
        default=1,
        flags=GObject.ParamFlags.READWRITE,
    )

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.buffer: List[int] = []

        self.last_tracks: List[int] = []
        self.last_coords: List[List[float]] = []
        self.last_mean_count: int = None
        self.room_people_counter: int = 0
        self.original_mask = read_mask()
        self._mask = None

    def get_mask(self, shape):
        shape = tuple(shape)
        if self._mask is not None and self._mask.shape == shape:
            return self._mask
        mask = cv2.resize(self.original_mask, shape)
        self._mask = np.clip(mask[:, :, 2], 0, 1)
        return self._mask

    def do_transform_caps(self, direction, caps, filter_):
        # (Kostya): without this override the element negotiates
        # the same formats on both caps, which is not what we need here:
        # our `src` cap must return predicts instead of a video stream.
        #
        # Copied from: https://github.com/GStreamer/gst-python/blob/30cc0fc83de306626bfb30b679156be30fbf1be9/examples/plugins/python/audioplot.py#L164

        if direction == Gst.PadDirection.SRC:
            res = ICAPS
        else:
            res = OCAPS

        if filter_:
            res = res.intersect(filter_)
        return res

    def do_transform(self, inbuf, outbuf):
        magic_header = 0xfa91ffff

        is_success, info = inbuf.map(Gst.MapFlags.READ)
        assert is_success is True

        s = '!LL'
        magic, payload_length, = struct.unpack_from(s, info.data)
        struct_size = struct.calcsize(s)
        assert magic == magic_header, f"unexpected payload! {magic:x}"
        shape, _labels, _scores, _coords, _tracks = msgpack.unpackb(
            info.data[struct_size: payload_length + struct_size],
            raw=False
        )

        inbuf.unmap(info)

        self.buffer.append(len(_labels))
        self.buffer = self.buffer[-self.max_age:]

        mean_count = int(sum(self.buffer)/len(self.buffer))

        mask = self.get_mask(shape)

        # check new tracks
        appeared = 0
        left = 0

        for idx, track in enumerate(_tracks):
            if track not in self.last_tracks:
                coords = _coords[idx]
                center_x = int((coords[0] + coords[2]) / 2)
                center_y = int((coords[1] + coords[3]) / 2)
                if mask[center_y, center_x]:
                    appeared += 1

        for idx, track in enumerate(self.last_tracks):
            if track not in _tracks:
                coords = self.last_coords[idx]
                center_x = int((coords[0] + coords[2]) / 2)
                center_y = int((coords[1] + coords[3]) / 2)
                if mask[center_y, center_x]:
                    left += 1

        self.last_tracks = _tracks
        self.last_coords = _coords

        room_count = 0xffffffff
        if self.last_mean_count is not None:
            delta = self.last_mean_count - mean_count
            self.room_people_counter += delta - left + appeared
            self.room_people_counter = max(0, self.room_people_counter)  # prevent subzero counts
            room_count = self.room_people_counter

        self.last_mean_count = mean_count

        meter_magic_header = 0xfa91f534
        full_data = struct.pack('!LLL', meter_magic_header, mean_count, room_count)

        assert len(full_data) <= outbuf.get_size(), (
            f"Too much data {len(full_data)} for buffer {outbuf.get_size()}"
        )

        outbuf.fill(0, full_data)

        return Gst.FlowReturn.OK

GObject.type_register(PeopleCounter)
__gstelementfactory__ = ("people_counter_py", Gst.Rank.NONE, PeopleCounter)


# -*- coding: utf-8 -*-
"""
Created on Wed Aug 22 10:59:07 2018

@author: 354651
"""

import tensorflow as tf
from model import model
import dataset

def main():
    with tf.Session() as sess:

        m = model(sess)

        sess.run(tf.global_variables_initializer())

        for epoch in range(10):
            total_cost = 0

            for step in range(dataset.total_batch):
                x_data, y_data = dataset.train_data()
                c, _ = m.train(x_data, y_data)
                total_cost += c

            print("{} cost: {}".format(epoch + 1, total_cost / dataset.total_batch))

        x_data, y_data = dataset.test_data()
        print("accuracy: {}".format(m.accuracy(x_data, y_data)))

if __name__ == '__main__':
    main()