# Distribuição Normal

Gaussiana, curva de sino

* simétrica
* média = mediana = moda
* variáveis contínuas

Ex:
* altura e peso de uma população
* tamanho do crânio de recém nascidos
* pressão sanguínea

$$ p(x|\mu,\sigma) = \frac{1}{\sqrt{2\pi\sigma^2}}\exp\left[-\frac{(x-\mu)^2}{2\sigma^2}\right] $$

```
import matplotlib.pyplot as plt

SMALL_SIZE = 12
MEDIUM_SIZE = 14
BIGGER_SIZE = 16

# Font Sizes
plt.rc('font', size=SMALL_SIZE)          # controls default text sizes
plt.rc('axes', titlesize=SMALL_SIZE)     # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE)    # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize
plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title

plt.rc('figure', figsize = (8, 6)) # Figure Size
```

```
import numpy as np
from scipy.stats import norm
import math

mu = 0
variance = 1
sigma = np.sqrt(variance)
x = np.linspace(mu - 3*sigma, mu + 3*sigma, 100)

p = 1/np.sqrt(2*math.pi*sigma**2)*np.exp(-(x-mu)**2/(2*sigma**2))

plt.plot(x, norm.pdf(x, mu, sigma), label='scipy')
plt.plot(x, p, 'k:', lw=5, label='formula')

plt.legend();
```

```
x = np.linspace(-5, 5, 100)

mu = 0
sigma = 1
plt.plot(x,norm.pdf(x, mu, sigma), 'k:', lw=5,
         label=r'$\mu={}, \sigma={}$'.format(mu,sigma))

mu = 1
sigma = 1.5
plt.plot(x,norm.pdf(x, mu, sigma), 'r-.', lw=2,
         label=r'$\mu={}, \sigma={}$'.format(mu,sigma))

mu = -1
sigma = .75
plt.plot(x,norm.pdf(x, mu, sigma), 'g', lw=3,
         label=r'$\mu={}, \sigma={}$'.format(mu,sigma))

plt.legend();
```

Z_score

$$ z_i = \frac{x_i-\bar{x}}{\sigma} $$

# Distribuição

* Variáveis Contínuas
 * Normal
 * Exponencial
 * Gamma

* Variáveis Discretas
 * Binomial
 * Poisson

# Distribuição binomial

* variáveis discretas
* dois resultados possíveis de igual chance de ocorrência
* eventos finitos e independentes

Ex.:
* Dado
* Moeda

$$ p(k|n,p) = \frac{n!}{k!(n-k)!}p^k(1-p)^{n-k} $$

```
from math import factorial
from scipy.stats import binom

n = 50
p = 0.5
k_vec = np.arange(1,n+1) # target, starts at 1 goes to n, all possible outcomes

def compute_binomial_prob(n,k,p):
  return factorial(n)/(factorial(k)*factorial(n-k)) * p**k * (1-p)**(n-k)

P_vec = [compute_binomial_prob(n,k,p) for k in k_vec]

plt.plot(k_vec, binom.pmf(k_vec, n, p), 'r', label='scipy')
plt.plot(k_vec, P_vec, 'k:', lw=5, label='formula')

plt.legend();

```

Ex: https://towardsdatascience.com/fun-with-the-binomial-distribution-96a5ecabf65b

```
# Import libraries
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
# Input variables
# Number of trials
trials = 1000
# Number of independent experiments in each trial
n = 10
# Probability of success for each experiment
p = 0.5
# Function that runs our coin toss trials (Monte Carlo)
# heads is a list of the number of successes from each trial of n experiments
def run_binom(trials, n, p):
    heads = []
    for i in range(trials):
        tosses = [np.random.random() for i in range(n)]
        heads.append(len([i for i in tosses if i>=0.50]))
    return heads
# Run the function
heads = run_binom(trials, n, p)
# Plot the results as a histogram
fig, ax = plt.subplots(figsize=(14,7))
ax = sns.distplot(heads, bins=11, label='simulation results')
ax.set_xlabel("Number of Heads",fontsize=16)
ax.set_ylabel("Frequency",fontsize=16);
```

```
# Plot the actual binomial distribution as a sanity check
from scipy.stats import binom
x = range(0,11)
plt.plot(x, binom.pmf(x, n, p), 'ro', label='actual binomial distribution')
plt.vlines(x, 0, binom.pmf(x, n, p), colors='r', lw=5, alpha=0.5)
plt.legend()
plt.show()
```

```
# Probability of getting 6 heads
runs = 10000
prob_6 = sum([1 for i in np.random.binomial(n, p, size=runs) if i==6])/runs
print('The probability of 6 heads is: ' + str(prob_6))
```

    The probability of 6 heads is: 0.201

```
# Call Center Simulation
# Number of employees to simulate
employees = 100
# Cost per employee
wage = 200
# Number of independent calls per employee
n = 50
# Probability of success for each call
p = 0.04
# Revenue per call
revenue = 100
# Binomial random variables of call center employees
conversions = np.random.binomial(n, p, size=employees)
# Print some key metrics of our call center
print('Average Conversions per Employee: ' + str(round(np.mean(conversions), 2)))
print('Standard Deviation of Conversions per Employee: ' + str(round(np.std(conversions), 2)))
print('Total Conversions: ' + str(np.sum(conversions)))
print('Total Revenues: ' + str(np.sum(conversions)*revenue))
print('Total Expense: ' + str(employees*wage))
print('Total Profits: ' + str(np.sum(conversions)*revenue - employees*wage))

# Number of days to simulate
sims = 1000
sim_conversions = [np.sum(np.random.binomial(n, p, size=employees)) for i in range(sims)]
sim_profits = np.array(sim_conversions)*revenue - employees*wage
```

    Average Conversions per Employee: 1.72
    Standard Deviation of Conversions per Employee: 1.28
    Total Conversions: 172
    Total Revenues: 17200
    Total Expense: 20000
    Total Profits: -2800

```
# Call Center Simulation (Higher Conversion Rate)
# Number of employees to simulate
employees = 100
# Cost per employee
wage = 200
# Number of independent calls per employee
n = 55
# Probability of success for each call
p = 0.05
# Revenue per call
revenue = 100
# Binomial random variables of call center employees
conversions_up = np.random.binomial(n, p, size=employees)
# Simulate 1,000 days for our call center
# Number of days to simulate
sims = 1000

sim_conversions_up = [np.sum(np.random.binomial(n, p, size=employees)) for i in range(sims)]
sim_profits_up = np.array(sim_conversions_up)*revenue - employees*wage
# Plot and save the results as a histogram
fig, ax = plt.subplots(figsize=(14,7))
ax = sns.distplot(sim_profits, bins=20, label='original call center simulation results')
ax = sns.distplot(sim_profits_up, bins=20, label='improved call center simulation results', color='red')
ax.set_xlabel("Profits",fontsize=16)
ax.set_ylabel("Frequency",fontsize=16)
plt.legend();
```

Ex: https://cmdlinetips.com/2018/03/probability-distributions-in-python/

https://www.probabilisticworld.com/discrete-probability-distributions-overview/

```
# for inline plots in jupyter
%matplotlib inline
# import matplotlib
import matplotlib.pyplot as plt
# import seaborn
import seaborn as sns
# settings for seaborn plotting style
sns.set(color_codes=True)
# settings for seaborn plot sizes
sns.set(rc={'figure.figsize':(4.5,3)})
```

# Uniform distribution

$$ P(x;n) = \frac{1}{n} $$

```
# import uniform distribution
from scipy.stats import uniform

# random numbers from uniform distribution
# Generate 10 numbers from 0 to 10
n = 10000
a = 0
b = 10
data_uniform = uniform.rvs(size=n, loc = a, scale=b) #random variables

ax = sns.distplot(data_uniform,
                  bins=100,
                  kde=False, # density plot
                  color='skyblue',
                  hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Uniform ', ylabel='Frequency');
```

# Normal distribution

$$ p(x|\mu,\sigma) = \frac{1}{\sqrt{2\pi\sigma^2}}\exp\left[-\frac{(x-\mu)^2}{2\sigma^2}\right] $$

```
from scipy.stats import norm

# generate random numbers from N(0,1)
data_normal = norm.rvs(size=10000,loc=0,scale=1)  # loc = mean, scale = std

ax = sns.distplot(data_normal,
                  bins=100,
                  kde=False,
                  color='skyblue',
                  hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Normal', ylabel='Frequency');
```

# Bernoulli distribution

Discrete. Outcome: 0 or 1.

\begin{equation}
P(x;p) = \begin{cases}
p &\text{if $x=1$}\\
1-p &\text{if $x=0$}
\end{cases}
\end{equation}

```
# import bernoulli
from scipy.stats import bernoulli

# generate bernoulli
data_bern = bernoulli.rvs(size=10000,p=0.3)
ax= sns.distplot(data_bern,
                 kde=False,
                 color="skyblue",
                 hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Bernoulli', ylabel='Frequency');

```

# Binomial distribution

Discrete.
Obtains the number of successe from N Bernoulli trials.

$$ p(k|n,p) = \frac{n!}{k!(n-k)!}p^k(1-p)^{n-k} $$

\begin{equation}
p(x; p,n) = \binom{n}{x}p^{x}(1-p)^{n-x}
\end{equation}

```
from scipy.stats import binom

binom.rvs(n=10,p=0.5) # sucesses from n=10 Bernoulli trials with p=0.5

data_binom = binom.rvs(n=10,p=0.5,size=10000)
ax = sns.distplot(data_binom,
                  kde=False,
                  color='skyblue',
                  hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Binomial', ylabel='Frequency');

```

# Poisson distribution

number of times an event happened in a time interval

* rate of ocurrence (mu)

$$ P(x;\mu)=\frac{\mu^x e^{-\mu}}{x!} $$

```
from scipy.stats import poisson

data_poisson = poisson.rvs(mu=3, size=10000)
ax = sns.distplot(data_poisson,
                  kde=False,
                  color='green',
                  hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Poisson', ylabel='Frequency');
```

# Beta distribution

* Continuous
* distribution for probabilities

```
from scipy.stats import beta

data_beta = beta.rvs(1, 1, size=10000) # ~ uniform
ax = sns.distplot(data_beta,
                  kde=False,
                  bins=100,
                  color='skyblue',
                  hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Beta(1,1)', ylabel='Frequency');
```

```
data_beta_a10b1 = beta.rvs(10, 1, size=10000) #skewed right
ax = sns.distplot(data_beta_a10b1,
             kde=False,
             bins=50,
             color='skyblue',
             hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Beta(10,1)', ylabel='Frequency');
```

```
data_beta_a1b10 = beta.rvs(1, 10, size=10000) # skewed leftt
ax = sns.distplot(data_beta_a1b10,
                kde=False,
                bins=100,
                color='skyblue',
                hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Beta(1,10)', ylabel='Frequency');
```

```
data_beta_a10b10 = beta.rvs(10, 10, size=10000) # ~ normal
ax = sns.distplot(data_beta_a10b10,
                  kde=False,
                  bins=100,
                  color='skyblue',
                  hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Beta(10,10)', ylabel='Frequency');
```

# Gamma distribution

```
from scipy.stats import gamma

data_gamma = gamma.rvs(a=5, size=10000)
ax = sns.distplot(data_gamma,
                  kde=False,
                  bins=100,
                  color='skyblue',
                  hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Gamma', ylabel='Frequency');
```

```
help(gamma)
```

    Help on gamma_gen in module scipy.stats._continuous_distns object:

    class gamma_gen(scipy.stats._distn_infrastructure.rv_continuous)
     |  A gamma continuous random variable.
     |
     |  %(before_notes)s
     |
     |  See Also
     |  --------
     |  erlang, expon
     |
     |  Notes
     |  -----
     |  The probability density function for `gamma` is:
     |
     |  .. math::
     |
     |      f(x, a) = \frac{x^{a-1} \exp(-x)}{\Gamma(a)}
     |
     |  for :math:`x \ge 0`, :math:`a > 0`. Here :math:`\Gamma(a)` refers to the
     |  gamma function.
     |
     |  `gamma` takes ``a`` as a shape parameter for :math:`a`.
     |
     |  When :math:`a` is an integer, `gamma` reduces to the Erlang
     |  distribution, and when :math:`a=1` to the exponential distribution.
     |
     |  %(after_notes)s
     |
     |  %(example)s
     |
     |  Method resolution order:
     |      gamma_gen
     |      scipy.stats._distn_infrastructure.rv_continuous
     |      scipy.stats._distn_infrastructure.rv_generic
     |      builtins.object
     |
     |  Methods defined here:
     |
     |  fit(self, data, *args, **kwds)
     |      Return MLEs for shape (if applicable), location, and scale
     |      parameters from data.
     |
     |      MLE stands for Maximum Likelihood Estimate.  Starting estimates for
     |      the fit are given by input arguments; for any arguments not provided
     |      with starting estimates, ``self._fitstart(data)`` is called to generate
     |      such.
     |
     |      One can hold some parameters fixed to specific values by passing in
     |      keyword arguments ``f0``, ``f1``, ..., ``fn`` (for shape parameters)
     |      and ``floc`` and ``fscale`` (for location and scale parameters,
     |      respectively).
     |
     |      Parameters
     |      ----------
     |      data : array_like
     |          Data to use in calculating the MLEs.
     |      args : floats, optional
     |          Starting value(s) for any shape-characterizing arguments (those not
     |          provided will be determined by a call to ``_fitstart(data)``).
     |          No default value.
     |      kwds : floats, optional
     |          Starting values for the location and scale parameters; no default.
     |          Special keyword arguments are recognized as holding certain
     |          parameters fixed:
     |
     |          - f0...fn : hold respective shape parameters fixed.
     |            Alternatively, shape parameters to fix can be specified by name.
     |            For example, if ``self.shapes == "a, b"``, ``fa``and ``fix_a``
     |            are equivalent to ``f0``, and ``fb`` and ``fix_b`` are
     |            equivalent to ``f1``.
     |
     |          - floc : hold location parameter fixed to specified value.
     |
     |          - fscale : hold scale parameter fixed to specified value.
     |
     |          - optimizer : The optimizer to use.  The optimizer must take ``func``,
     |            and starting position as the first two arguments,
     |            plus ``args`` (for extra arguments to pass to the
     |            function to be optimized) and ``disp=0`` to suppress
     |            output as keyword arguments.
     |
     |      Returns
     |      -------
     |      mle_tuple : tuple of floats
     |          MLEs for any shape parameters (if applicable), followed by those
     |          for location and scale. For most random variables, shape statistics
     |          will be returned, but there are exceptions (e.g. ``norm``).
     |
     |      Notes
     |      -----
     |      This fit is computed by maximizing a log-likelihood function, with
     |      penalty applied for samples outside of range of the distribution. The
     |      returned answer is not guaranteed to be the globally optimal MLE, it
     |      may only be locally optimal, or the optimization may fail altogether.
     |      If the data contain any of np.nan, np.inf, or -np.inf, the fit routine
     |      will throw a RuntimeError.
     |
     |      When the location is fixed by using the argument `floc`, this
     |      function uses explicit formulas or solves a simpler numerical
     |      problem than the full ML optimization problem.  So in that case,
     |      the `optimizer`, `loc` and `scale` arguments are ignored.
     |
     |      Examples
     |      --------
     |
     |      Generate some data to fit: draw random variates from the `beta`
     |      distribution
     |
     |      >>> from scipy.stats import beta
     |      >>> a, b = 1., 2.
     |      >>> x = beta.rvs(a, b, size=1000)
     |
     |      Now we can fit all four parameters (``a``, ``b``, ``loc`` and ``scale``):
     |
     |      >>> a1, b1, loc1, scale1 = beta.fit(x)
     |
     |      We can also use some prior knowledge about the dataset: let's keep
     |      ``loc`` and ``scale`` fixed:
     |
     |      >>> a1, b1, loc1, scale1 = beta.fit(x, floc=0, fscale=1)
     |      >>> loc1, scale1
     |      (0, 1)
     |
     |      We can also keep shape parameters fixed by using ``f``-keywords. To
     |      keep the zero-th shape parameter ``a`` equal 1, use ``f0=1`` or,
     |      equivalently, ``fa=1``:
     |
     |      >>> a1, b1, loc1, scale1 = beta.fit(x, fa=1, floc=0, fscale=1)
     |      >>> a1
     |      1
     |
     |      Not all distributions return estimates for the shape parameters.
     |      ``norm`` for example just returns estimates for location and scale:
     |
     |      >>> from scipy.stats import norm
     |      >>> x = norm.rvs(a, b, size=1000, random_state=123)
     |      >>> loc1, scale1 = norm.fit(x)
     |      >>> loc1, scale1
     |      (0.92087172783841631, 2.0015750750324668)
     |
     |  ----------------------------------------------------------------------
     |  Methods inherited from scipy.stats._distn_infrastructure.rv_continuous:
     |
     |  __init__(self, momtype=1, a=None, b=None, xtol=1e-14, badvalue=None, name=None, longname=None, shapes=None, extradoc=None, seed=None)
     |      Initialize self.  See help(type(self)) for accurate signature.
     |
     |  cdf(self, x, *args, **kwds)
     |      Cumulative distribution function of the given RV.
     |
     |      Parameters
     |      ----------
     |      x : array_like
     |          quantiles
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      cdf : ndarray
     |          Cumulative distribution function evaluated at `x`
     |
     |  expect(self, func=None, args=(), loc=0, scale=1, lb=None, ub=None, conditional=False, **kwds)
     |      Calculate expected value of a function with respect to the
     |      distribution by numerical integration.
     |
     |      The expected value of a function ``f(x)`` with respect to a
     |      distribution ``dist`` is defined as::
     |
     |                  ub
     |          E[f(x)] = Integral(f(x) * dist.pdf(x)),
     |                  lb
     |
     |      where ``ub`` and ``lb`` are arguments and ``x`` has the ``dist.pdf(x)``
     |      distribution. If the bounds ``lb`` and ``ub`` correspond to the
     |      support of the distribution, e.g. ``[-inf, inf]`` in the default
     |      case, then the integral is the unrestricted expectation of ``f(x)``.
     |      Also, the function ``f(x)`` may be defined such that ``f(x)`` is ``0``
     |      outside a finite interval in which case the expectation is
     |      calculated within the finite range ``[lb, ub]``.
     |
     |      Parameters
     |      ----------
     |      func : callable, optional
     |          Function for which integral is calculated. Takes only one argument.
     |          The default is the identity mapping f(x) = x.
     |      args : tuple, optional
     |          Shape parameters of the distribution.
     |      loc : float, optional
     |          Location parameter (default=0).
     |      scale : float, optional
     |          Scale parameter (default=1).
     |      lb, ub : scalar, optional
     |          Lower and upper bound for integration. Default is set to the
     |          support of the distribution.
     |      conditional : bool, optional
     |          If True, the integral is corrected by the conditional probability
     |          of the integration interval.  The return value is the expectation
     |          of the function, conditional on being in the given interval.
     |          Default is False.
     |
     |      Additional keyword arguments are passed to the integration routine.
     |
     |      Returns
     |      -------
     |      expect : float
     |          The calculated expected value.
     |
     |      Notes
     |      -----
     |      The integration behavior of this function is inherited from
     |      `scipy.integrate.quad`. Neither this function nor
     |      `scipy.integrate.quad` can verify whether the integral exists or is
     |      finite. For example ``cauchy(0).mean()`` returns ``np.nan`` and
     |      ``cauchy(0).expect()`` returns ``0.0``.
     |
     |      Examples
     |      --------
     |
     |      To understand the effect of the bounds of integration consider
     |
     |      >>> from scipy.stats import expon
     |      >>> expon(1).expect(lambda x: 1, lb=0.0, ub=2.0)
     |      0.6321205588285578
     |
     |      This is close to
     |
     |      >>> expon(1).cdf(2.0) - expon(1).cdf(0.0)
     |      0.6321205588285577
     |
     |      If ``conditional=True``
     |
     |      >>> expon(1).expect(lambda x: 1, lb=0.0, ub=2.0, conditional=True)
     |      1.0000000000000002
     |
     |      The slight deviation from 1 is due to numerical integration.
     |
     |  fit_loc_scale(self, data, *args)
     |      Estimate loc and scale parameters from data using 1st and 2nd moments.
     |
     |      Parameters
     |      ----------
     |      data : array_like
     |          Data to fit.
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information).
     |
     |      Returns
     |      -------
     |      Lhat : float
     |          Estimated location parameter for the data.
     |      Shat : float
     |          Estimated scale parameter for the data.
     |
     |  isf(self, q, *args, **kwds)
     |      Inverse survival function (inverse of `sf`) at q of the given RV.
     |
     |      Parameters
     |      ----------
     |      q : array_like
     |          upper tail probability
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      x : ndarray or scalar
     |          Quantile corresponding to the upper tail probability q.
     |
     |  logcdf(self, x, *args, **kwds)
     |      Log of the cumulative distribution function at x of the given RV.
     |
     |      Parameters
     |      ----------
     |      x : array_like
     |          quantiles
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      logcdf : array_like
     |          Log of the cumulative distribution function evaluated at x
     |
     |  logpdf(self, x, *args, **kwds)
     |      Log of the probability density function at x of the given RV.
     |
     |      This uses a more numerically accurate calculation if available.
     |
     |      Parameters
     |      ----------
     |      x : array_like
     |          quantiles
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      logpdf : array_like
     |          Log of the probability density function evaluated at x
     |
     |  logsf(self, x, *args, **kwds)
     |      Log of the survival function of the given RV.
     |
     |      Returns the log of the "survival function," defined as (1 - `cdf`),
     |      evaluated at `x`.
     |
     |      Parameters
     |      ----------
     |      x : array_like
     |          quantiles
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      logsf : ndarray
     |          Log of the survival function evaluated at `x`.
     |
     |  nnlf(self, theta, x)
     |      Return negative loglikelihood function.
     |
     |      Notes
     |      -----
     |      This is ``-sum(log pdf(x, theta), axis=0)`` where `theta` are the
     |      parameters (including loc and scale).
     |
     |  pdf(self, x, *args, **kwds)
     |      Probability density function at x of the given RV.
     |
     |      Parameters
     |      ----------
     |      x : array_like
     |          quantiles
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      pdf : ndarray
     |          Probability density function evaluated at x
     |
     |  ppf(self, q, *args, **kwds)
     |      Percent point function (inverse of `cdf`) at q of the given RV.
     |
     |      Parameters
     |      ----------
     |      q : array_like
     |          lower tail probability
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      x : array_like
     |          quantile corresponding to the lower tail probability q.
     |
     |  sf(self, x, *args, **kwds)
     |      Survival function (1 - `cdf`) at x of the given RV.
     |
     |      Parameters
     |      ----------
     |      x : array_like
     |          quantiles
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      sf : array_like
     |          Survival function evaluated at x
     |
     |  ----------------------------------------------------------------------
     |  Methods inherited from scipy.stats._distn_infrastructure.rv_generic:
     |
     |  __call__(self, *args, **kwds)
     |      Freeze the distribution for the given arguments.
     |
     |      Parameters
     |      ----------
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution.  Should include all
     |          the non-optional arguments, may include ``loc`` and ``scale``.
     |
     |      Returns
     |      -------
     |      rv_frozen : rv_frozen instance
     |          The frozen distribution.
     |
     |  __getstate__(self)
     |
     |  __setstate__(self, state)
     |
     |  entropy(self, *args, **kwds)
     |      Differential entropy of the RV.
     |
     |      Parameters
     |      ----------
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information).
     |      loc : array_like, optional
     |          Location parameter (default=0).
     |      scale : array_like, optional  (continuous distributions only).
     |          Scale parameter (default=1).
     |
     |      Notes
     |      -----
     |      Entropy is defined base `e`:
     |
     |      >>> drv = rv_discrete(values=((0, 1), (0.5, 0.5)))
     |      >>> np.allclose(drv.entropy(), np.log(2.0))
     |      True
     |
     |  freeze(self, *args, **kwds)
     |      Freeze the distribution for the given arguments.
     |
     |      Parameters
     |      ----------
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution.  Should include all
     |          the non-optional arguments, may include ``loc`` and ``scale``.
     |
     |      Returns
     |      -------
     |      rv_frozen : rv_frozen instance
     |          The frozen distribution.
     |
     |  interval(self, alpha, *args, **kwds)
     |      Confidence interval with equal areas around the median.
     |
     |      Parameters
     |      ----------
     |      alpha : array_like of float
     |          Probability that an rv will be drawn from the returned range.
     |          Each value should be in the range [0, 1].
     |      arg1, arg2, ... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information).
     |      loc : array_like, optional
     |          location parameter, Default is 0.
     |      scale : array_like, optional
     |          scale parameter, Default is 1.
     |
     |      Returns
     |      -------
     |      a, b : ndarray of float
     |          end-points of range that contain ``100 * alpha %`` of the rv's
     |          possible values.
     |
     |  mean(self, *args, **kwds)
     |      Mean of the distribution.
     |
     |      Parameters
     |      ----------
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      mean : float
     |          the mean of the distribution
     |
     |  median(self, *args, **kwds)
     |      Median of the distribution.
     |
     |      Parameters
     |      ----------
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          Location parameter, Default is 0.
     |      scale : array_like, optional
     |          Scale parameter, Default is 1.
     |
     |      Returns
     |      -------
     |      median : float
     |          The median of the distribution.
     |
     |      See Also
     |      --------
     |      rv_discrete.ppf
     |          Inverse of the CDF
     |
     |  moment(self, n, *args, **kwds)
     |      n-th order non-central moment of distribution.
     |
     |      Parameters
     |      ----------
     |      n : int, n >= 1
     |          Order of moment.
     |      arg1, arg2, arg3,... : float
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information).
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |  rvs(self, *args, **kwds)
     |      Random variates of given type.
     |
     |      Parameters
     |      ----------
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information).
     |      loc : array_like, optional
     |          Location parameter (default=0).
     |      scale : array_like, optional
     |          Scale parameter (default=1).
     |      size : int or tuple of ints, optional
     |          Defining number of random variates (default is 1).
     |      random_state : None or int or ``np.random.RandomState`` instance, optional
     |          If int or RandomState, use it for drawing the random variates.
     |          If None, rely on ``self.random_state``.
     |          Default is None.
     |
     |      Returns
     |      -------
     |      rvs : ndarray or scalar
     |          Random variates of given `size`.
     |
     |  stats(self, *args, **kwds)
     |      Some statistics of the given RV.
     |
     |      Parameters
     |      ----------
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional (continuous RVs only)
     |          scale parameter (default=1)
     |      moments : str, optional
     |          composed of letters ['mvsk'] defining which moments to compute:
     |          'm' = mean,
     |          'v' = variance,
     |          's' = (Fisher's) skew,
     |          'k' = (Fisher's) kurtosis.
     |          (default is 'mv')
     |
     |      Returns
     |      -------
     |      stats : sequence
     |          of requested moments.
     |
     |  std(self, *args, **kwds)
     |      Standard deviation of the distribution.
     |
     |      Parameters
     |      ----------
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      std : float
     |          standard deviation of the distribution
     |
     |  support(self, *args, **kwargs)
     |      Return the support of the distribution.
     |
     |      Parameters
     |      ----------
     |      arg1, arg2, ... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information).
     |      loc : array_like, optional
     |          location parameter, Default is 0.
     |      scale : array_like, optional
     |          scale parameter, Default is 1.
     |      Returns
     |      -------
     |      a, b : float
     |          end-points of the distribution's support.
     |
     |  var(self, *args, **kwds)
     |      Variance of the distribution.
     |
     |      Parameters
     |      ----------
     |      arg1, arg2, arg3,... : array_like
     |          The shape parameter(s) for the distribution (see docstring of the
     |          instance object for more information)
     |      loc : array_like, optional
     |          location parameter (default=0)
     |      scale : array_like, optional
     |          scale parameter (default=1)
     |
     |      Returns
     |      -------
     |      var : float
     |          the variance of the distribution
     |
     |  ----------------------------------------------------------------------
     |  Data descriptors inherited from scipy.stats._distn_infrastructure.rv_generic:
     |
     |  __dict__
     |      dictionary for instance variables (if defined)
     |
     |  __weakref__
     |      list of weak references to the object (if defined)
     |
     |  random_state
     |      Get or set the RandomState object for generating random variates.
     |
     |      This can be either None or an existing RandomState object.
     |
     |      If None (or np.random), use the RandomState singleton used by np.random.
     |      If already a RandomState instance, use it.
     |      If an int, use a new RandomState instance seeded with seed.

# Lognormal distribution

```
from scipy.stats import lognorm

data_lognorm = lognorm.rvs(0.2, size=10000)
ax = sns.distplot(data_lognorm,kde=False,
                  bins=100,
                  color='skyblue',
                  hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Log Normal', ylabel='Frequency');
```

# Negative Binomial distribution
* discrete

```
from scipy.stats import nbinom

data_nbinom = nbinom.rvs(10, 0.5, size=10000)
ax = sns.distplot(data_nbinom,
                  kde=False,
                  color='skyblue',
                  hist_kws={"linewidth": 15,'alpha':1})
ax.set(xlabel='Negative Binomial', ylabel='Frequency');
```

# PDF e CDF
PDF: Função Densidade de Probabilidade
* relativa
* não negativa
* integral = 1

CDF: Função Distribuição Acumulada
* acumulada
* não negativa
* integral = 1

Survivor fuction = 1 - CDF

ECDF
* estimador empírico

```

```


## Basic sympy setting

```python
%reset -f
from sympy import *    # import everything from sympy module
init_printing()        # for nice math output

## forcing plots inside browser
%matplotlib inline
```

###  Declare symbolic variables

```python
x = symbols('x')
```

### Example 1

Find 4-th degree Taylor polynamial of $f(x) = \sin(x)$ at $a = \frac{\pi}{2}$. Plot $f(x)$ and Taylor polynomial on the same window

```python
## Finding Taylor polynomial

a = pi/2
f = sin(x)

T = f.subs(x,a) \
    + diff(f,x,1).subs(x,a)*(x-a)  \
    + diff(f,x,2).subs(x,a)*(x-a)**2/2 \
    + diff(f,x,3).subs(x,a)*(x-a)**3/factorial(3) \
    + diff(f,x,4).subs(x,a)*(x-a)**4/factorial(4)

T

```

```python
## plotting f(x) and T(x) in same window

p = plot(f,T, (x,a - 4, a + 4),show=False, legend=True)

p[0].line_color = 'blue'
p[1].line_color = 'green'

p[0].label      = 'f(x)'
p[1].label      = 'T(x)'

p.show()
```

### Example 2

Find 20-th degree Taylor polynamial of $f(x) = \log(x)$ at $a = 1$. Plot $f(x)$ and Taylor polynomial on the same window

```python
## evaluation of T

a = 1
n = 20
f = log(x)

T = f.subs(x,a)

for k in range(1,n+1):
    df = diff(f,x,k)
    T = T + df.subs(x,a)*(x-a)**k/factorial(k)

T
```

```python
## plotting f(x) and T(x) in same window

p = plot(f,T, (x, 0.5, 2),show=False, legend=True)

p[0].line_color = 'blue'
p[1].line_color = 'green'

p[0].label      = 'f(x)'
p[1].label      = 'T(x)'

p.show()
```

```python

```


## Computing a running variance the smart way

Suppose we compute the running mean of a list of numbers $x_1,\,x_2,\,...,x_p,...$ when the $p^{th}$ number $x_p$ arrives the *smart way* (as shown above)

\begin{equation}
h_{p}^{\text{ave}} = \frac{p-1}{p}h_{p-1}^{\text{ave}} + \frac{1}{p}x_p
\end{equation}

and then - on top of this - we would like to compute the *running variance* of our ever-increasing list of numbers.  If we used the standard formula for variance we would compute

\begin{equation}
h_p^{\text{var}} = \frac{1}{p}\sum_{j=1}^p\left(x_j - h_p^{\text{ave}}\right)^2
\end{equation}

each time a new point $x_p$ arrived.  However computing the running variance this way would be wasteful - both in terms of storage (we would need to store all of the previous input points) and computation (we're repeating the same sorts of computation over and over again) - in complete analogy to the use of the standard formula when computing the running mean (as we saw above).

Multiplying both sides of Equation (2) by $p$ we have

\begin{equation}
p\,h_p^{\text{var}} = \sum_{j=1}^p\left(x_j - h_p^{\text{ave}}\right)^2
\end{equation}

Plugging in $p-1$ in place of $p$ in the Equation above yields

\begin{equation}
\left(p-1\right)\,h_{p-1}^{\text{var}} = \sum_{j=1}^{p-1}\left(x_j - h_{p-1}^{\text{ave}}\right)^2
\end{equation}

Subtracting (4) from (3), and simple re-arrangement of terms gives

\begin{equation}
\begin{split}
p\,h_p^{\text{var}} - \left(p-1\right)\,h_{p-1}^{\text{var}} & =  \sum_{j=1}^p\left(x_j - h_p^{\text{ave}}\right)^2 - \sum_{j=1}^{p-1}\left(x_j - h_{p-1}^{\text{ave}}\right)^2 \\
& = \left[\left(x_p - h_p^{\text{ave}}\right)^2 + \sum_{j=1}^{p-1}\left(x_j - h_p^{\text{ave}}\right)^2 \right] - \sum_{j=1}^{p-1}\left(x_j - h_{p-1}^{\text{ave}}\right)^2 \\
& = \left(x_p - h_p^{\text{ave}}\right)^2 + \left[\sum_{j=1}^{p-1}\left(x_j - h_p^{\text{ave}}\right)^2 - \sum_{j=1}^{p-1}\left(x_j - h_{p-1}^{\text{ave}}\right)^2\right] \\
& = \left(x_p - h_p^{\text{ave}}\right)^2 + \sum_{j=1}^{p-1}\left[\left(x_j - h_p^{\text{ave}}\right)^2 - \left(x_j - h_{p-1}^{\text{ave}}\right)^2\right] \\
\end{split}
\end{equation}

We now use the identity $a^2 - b^2 = \left(a-b\right)\left(a+b\right)$ to simplify each summand in Equation (5) as follows

\begin{equation}
\begin{split}
p\,h_p^{\text{var}} - \left(p-1\right)\,h_{p-1}^{\text{var}} & = \left(x_p - h_p^{\text{ave}}\right)^2 + \sum_{j=1}^{p-1}\left(h_{p-1}^{\text{ave}} - h_p^{\text{ave}}\right) \left(2 x_j - h_{p}^{\text{ave}}-h_{p-1}^{\text{ave}}\right) \\
 & = \left(x_p - h_p^{\text{ave}}\right)^2 + \left(h_{p-1}^{\text{ave}} - h_p^{\text{ave}}\right) \sum_{j=1}^{p-1} \left(2 x_j - h_{p}^{\text{ave}}-h_{p-1}^{\text{ave}}\right)\\
\end{split}
\end{equation}

Notice, the last summation in Equation above can be written equivalently as

\begin{equation}
\begin{split}
\sum_{j=1}^{p-1} \left(2 x_j - h_{p}^{\text{ave}}-h_{p-1}^{\text{ave}}\right) & = \left(2\sum_{j=1}^{p-1} x_j\right) - \left(p-1\right) \left( h_{p}^{\text{ave}} + h_{p-1}^{\text{ave}} \right)\\
& = 2\left(p-1\right)h_{p-1}^{\text{ave}} - \left(p-1\right) \left( h_{p}^{\text{ave}} + h_{p-1}^{\text{ave}} \right)\\
& = \left(p-1\right) h_{p-1}^{\text{ave}} - \left(p-1\right) h_{p}^{\text{ave}} \\
& = \left(p \,h_{p}^{\text{ave}} - x_p\right) - \left(p-1\right) h_{p}^{\text{ave}} \\
& = h_{p}^{\text{ave}} - x_p
\end{split}
\end{equation}

where we have made use of the fact that $\left(p-1\right) h_{p-1}^{\text{ave}} = p \,h_{p}^{\text{ave}} - x_p$.

Substituting the result above into (6) we now have

\begin{equation}
\begin{split}
p\,h_p^{\text{var}} - \left(p-1\right)\,h_{p-1}^{\text{var}} & = \left(x_p - h_p^{\text{ave}}\right)^2 + \left(h_{p-1}^{\text{ave}} - h_p^{\text{ave}}\right)  \left(h_{p}^{\text{ave}} - x_p\right)\\
 & = \left(x_p - h_p^{\text{ave}}\right)\left(x_p - h_{p-1}^{\text{ave}}\right)
\end{split}
\end{equation}

A little re-arrangement finally gives

\begin{equation}
h_{p}^{\text{var}} = \frac{p-1}{p}h_{p-1}^{\text{var}} + \frac{1}{p}\left(x_p^{\,} - h_{p}^{\text{ave}}\right)\left(x_p^{\,} - h_{p-1}^{\text{ave}}\right)
\end{equation}


```python
OABC is any quadrilateral in 3D space.
P is the midpoint of OA, Q is the midpoint of AB, R is the midpoint of BC and S is the midpoint of OC.
Prove that PQ is parallel to SR
```

```python
#Define a coordinate system
from sympy.vector import CoordSys3D
Sys = CoordSys3D('Sys')
```

```python
# Define point O to be Sys’ origin. We can do this without loss of generality
O = Sys.origin

```

```python
#Define point A with respect to O

from sympy import symbols
a1, a2, a3 = symbols('a1 a2 a3')
A = O.locate_new('A', a1*Sys.i + a2*Sys.j + a3*Sys.k)
```

$\displaystyle Point\left(A, (a_{1})\mathbf{\hat{i}_{Sys}} + (a_{2})\mathbf{\hat{j}_{Sys}} + (a_{3})\mathbf{\hat{k}_{Sys}}, Point\left(Sys.origin, \mathbf{\hat{0}}\right)\right)$

```python
A
```

$\displaystyle Point\left(A, (a_{1})\mathbf{\hat{i}_{Sys}} + (a_{2})\mathbf{\hat{j}_{Sys}} + (a_{3})\mathbf{\hat{k}_{Sys}}, Point\left(Sys.origin, \mathbf{\hat{0}}\right)\right)$

```python
# Similarly define points B and C

b1, b2, b3 = symbols('b1 b2 b3')
B = O.locate_new('B', b1*Sys.i + b2*Sys.j + b3*Sys.k)
c1, c2, c3 = symbols('c1 c2 c3')
C = O.locate_new('C', c1*Sys.i + c2*Sys.j + c3*Sys.k)
```

```python
B
```

$\displaystyle Point\left(B, (b_{1})\mathbf{\hat{i}_{Sys}} + (b_{2})\mathbf{\hat{j}_{Sys}} + (b_{3})\mathbf{\hat{k}_{Sys}}, Point\left(Sys.origin, \mathbf{\hat{0}}\right)\right)$

```python
C
```

$\displaystyle Point\left(C, (c_{1})\mathbf{\hat{i}_{Sys}} + (c_{2})\mathbf{\hat{j}_{Sys}} + (c_{3})\mathbf{\hat{k}_{Sys}}, Point\left(Sys.origin, \mathbf{\hat{0}}\right)\right)$

```python
# P is the midpoint of OA. Lets locate it with respect to O (you could also define it with respect to A).
P = O.locate_new('P', A.position_wrt(O) + (O.position_wrt(A) / 2))
P
```

$\displaystyle Point\left(P, (\frac{a_{1}}{2})\mathbf{\hat{i}_{Sys}} + (\frac{a_{2}}{2})\mathbf{\hat{j}_{Sys}} + (\frac{a_{3}}{2})\mathbf{\hat{k}_{Sys}}, Point\left(Sys.origin, \mathbf{\hat{0}}\right)\right)$

```python
# Similarly define points Q, R and S as per the problem definitions.
Q = A.locate_new('Q', B.position_wrt(A) / 2)
R = B.locate_new('R', C.position_wrt(B) / 2)
S = O.locate_new('R', C.position_wrt(O) / 2)
```

```python
Q
```

$\displaystyle Point\left(Q, (- \frac{a_{1}}{2} + \frac{b_{1}}{2})\mathbf{\hat{i}_{Sys}} + (- \frac{a_{2}}{2} + \frac{b_{2}}{2})\mathbf{\hat{j}_{Sys}} + (- \frac{a_{3}}{2} + \frac{b_{3}}{2})\mathbf{\hat{k}_{Sys}}, Point\left(A, (a_{1})\mathbf{\hat{i}_{Sys}} + (a_{2})\mathbf{\hat{j}_{Sys}} + (a_{3})\mathbf{\hat{k}_{Sys}}, Point\left(Sys.origin, \mathbf{\hat{0}}\right)\right)\right)$

```python
R
```

$\displaystyle Point\left(R, (- \frac{b_{1}}{2} + \frac{c_{1}}{2})\mathbf{\hat{i}_{Sys}} + (- \frac{b_{2}}{2} + \frac{c_{2}}{2})\mathbf{\hat{j}_{Sys}} + (- \frac{b_{3}}{2} + \frac{c_{3}}{2})\mathbf{\hat{k}_{Sys}}, Point\left(B, (b_{1})\mathbf{\hat{i}_{Sys}} + (b_{2})\mathbf{\hat{j}_{Sys}} + (b_{3})\mathbf{\hat{k}_{Sys}}, Point\left(Sys.origin, \mathbf{\hat{0}}\right)\right)\right)$

```python
S
```

$\displaystyle Point\left(R, (\frac{c_{1}}{2})\mathbf{\hat{i}_{Sys}} + (\frac{c_{2}}{2})\mathbf{\hat{j}_{Sys}} + (\frac{c_{3}}{2})\mathbf{\hat{k}_{Sys}}, Point\left(Sys.origin, \mathbf{\hat{0}}\right)\right)$

```python
# Now compute the vectors in the directions specified by PQ and SR.
PQ = Q.position_wrt(P)
SR = R.position_wrt(S)
```

$\displaystyle (\frac{b_{1}}{2})\mathbf{\hat{i}_{Sys}} + (\frac{b_{2}}{2})\mathbf{\hat{j}_{Sys}} + (\frac{b_{3}}{2})\mathbf{\hat{k}_{Sys}}$

```python
PQ
```

$\displaystyle (\frac{b_{1}}{2})\mathbf{\hat{i}_{Sys}} + (\frac{b_{2}}{2})\mathbf{\hat{j}_{Sys}} + (\frac{b_{3}}{2})\mathbf{\hat{k}_{Sys}}$

```python
SR
```

$\displaystyle (\frac{b_{1}}{2})\mathbf{\hat{i}_{Sys}} + (\frac{b_{2}}{2})\mathbf{\hat{j}_{Sys}} + (\frac{b_{3}}{2})\mathbf{\hat{k}_{Sys}}$

```python
# Compute cross product
PQ.cross(SR)
```

$\displaystyle \mathbf{\hat{0}}$

```python
# Since the cross product is a zero vector, the two vectors have to be parallel, thus proving that PQ || SR.
```

Personal_Projects/vector math calculus 1.ipynb


# Exercise 2: Sinusoids and the DFT

Doing this exercise you will get a better understanding of the basics elements and operations that take place in the Discrete Fourier Transform (DFT). There are five parts: 1) Generate a sinusoid, 2) Generate a complex sinusoid, 3) Implement the DFT, 4) Implement the IDFT, and 5) Compute the magnitude spectrum of an input sequence.

### Relevant Concepts

A real sinusoid in discrete time domain can be expressed by:

\begin{equation}
x[n] = A\cos(2 \pi fnT + \varphi)
\end{equation}

where, $x$ is the array of real values of the sinusoid, $n$ is an integer value expressing the time index, $A$ is the amplitude value of the sinusoid, $f$ is the frequency value of the sinusoid in Hz, $T$ is the sampling period equal to $1/fs$, fs is the sampling frequency in Hz, and $\varphi$ is the initial phase of the sinusoid in radians.

A complex sinusoid in discrete time domain can be expressed by:

\begin{equation}
\bar{x}[n] = Ae^{j(\omega nT + \varphi)} = A\cos(\omega nT + \varphi)+ j A\sin(\omega nT + \varphi)
\end{equation}

where, $\bar{x}$ is the array of complex values of the sinusoid, $n$ is an integer value expressing the time index, $A$ is the amplitude value of the sinusoid, $e$ is the complex exponential number, $\omega$ is the frequency of the sinusoid in radians per second (equal to $2 \pi f$), $T$ is the sampling period equal $1/fs$, fs is the sampling frequency in Hz and $\varphi$ is the initial phase of the sinusoid in radians.

The $N$ point DFT of a sequence of real values $x$ (a sound) can be expressed by:

\begin{equation}
X[k] = \sum_{n=0}^{N-1} x[n]e^{-j2 \pi kn/N}  \hspace{1cm} k=0,...,N-1
\end{equation}

where $n$ is an integer value expressing the discrete time index, $k$ is an integer value expressing the discrete frequency index, and $N$ is the length of the DFT.

The IDFT of a spectrum $X$ of length $N$ can be expressed by:

\begin{equation}
x[n] = \frac{1}{N} \sum_{k=0}^{N-1} X[k]e^{j2 \pi kn/N}  \hspace{1cm} n=0,...,N-1
\end{equation}

where, $n$ is an integer value expressing the discrete time index, $k$ is an integer value expressing the discrete frequency index, and $N$ is the length of the spectrum $X$.

The magnitude of a complex spectrum $X$ is obtained by taking its absolute value: $|X[k]| $

## Part 1 - Generate a sinusoid

Complete the function `gen_sine` to generate a real sinusoid (use `np.cos()`) given its amplitude `A`, frequency `f` (Hz), initial phase `phi` (radians), sampling rate `fs` (Hz) and duration `t` (seconds).

All the input arguments to this function `(A, f, phi, fs and t)` are real numbers such that `A`, `t` and `fs` are positive, and `fs > 2*f` to avoid aliasing. The function should return a numpy array `x` of the generated sinusoid.

Use the function `cos` of the numpy package to compute the sinusoidal values.

```python
import numpy as np
```

```python
def gen_sine(A, f, phi, fs, t):
    """
    Inputs:
        A (float) =  amplitude of the sinusoid
        f (float) = frequency of the sinusoid in Hz
        phi (float) = initial phase of the sinusoid in radians
        fs (float) = sampling frequency of the sinusoid in Hz
        t (float) =  duration of the sinusoid (is second)
    Output:
        x (numpy array) = generated sinusoid
    """
    ### your code here

```

Now call and test the `gen_sine()` function. If you use `A=1.0, f = 10.0, phi = 1.0, fs = 50` and `t = 0.1`, the output numpy array should be:

```
array([ 0.54030231, -0.63332387, -0.93171798,  0.05749049,  0.96724906])
```

To generate a sinewave that you can hear, it should be longer and with a higher sampling rate. For example you can use `A=1.0, f = 440.0, phi = 1.0, fs = 5000` and `t = 0.5`. To play it import the `Ipython.display` package and use `ipd.display(ipd.Audio(data=x, rate=fs))`.

```python
import IPython.display as ipd

### your code here

```

## Part 2 - Generate a complex sinusoid

Complete the `gen_complex_sine()` function to generate the complex sinusoid that is used in DFT computation of length `N` (samples), corresponding to the frequency index `k`. Note that the complex sinusoid used in DFT computation has a negative sign in the exponential function.

The amplitude of such a complex sinusoid is `1`, the length is `N`, and the frequency in radians is `2*pi*k/N`.

The input arguments to the function are two positive integers, `k` and `N`, such that `k < N-1`. The function should return `c_sine`, a numpy array of the complex sinusoid. Use the function `exp()` of the numpy package to compute the complex sinusoidal values.

```python
def gen_complex_sine(k, N):
    """
    Inputs:
        k (integer) = frequency index of the complex sinusoid of the DFT
        N (integer) = length of complex sinusoid in samples
    Output:
        c_sine (numpy array) = generated complex sinusoid (length N)
    """
    ### your code here

```

Now run an test the `gen_complex_sine()` function. If you run it using `k=1` and `N=5`, the function should return the following numpy array:

```
array([ 1.0 + 0.j,  0.30901699 - 0.95105652j, -0.80901699 - 0.58778525j, -0.80901699 + 0.58778525j, 0.30901699 + 0.95105652j])
```

```python
# call gen_complex_sine
### your code here

```

## Part 3 - Implement the discrete Fourier transform (DFT)

Complete the function `dft()` to implement the discrete Fourier transform (DFT) equation given above. Given a sequence `x` of length `N`, the function should return its spectrum of length `N` with the frequency indexes ranging from 0 to `N-1`.

The input argument to the function is a numpy array `x` and the function should return a numpy array `X`, the DFT of `x`.

```python
def dft(x):
    """
    Input:
        x (numpy array) = input sequence of length N
    Output:
        X (numpy array) = N point DFT of the input sequence x
    """
    ## Your code here

```

Now run and test the function `dft()`. If you run it using as input `x = np.array([1, 2, 3, 4])`, the function shoulds return the following numpy array:

```
array([10.0 + 0.0j,  -2. +2.0j,  -2.0 - 9.79717439e-16j, -2.0 - 2.0j])
```

Note that you might not get an exact 0 in the output because of the small numerical errors due to the limited precision of the data in your computer. Usually these errors are of the order 1e-15 depending on your machine.

```python
# call dft
### your code here

```

## Part 4 - Implement the inverse discrete Fourier transform (IDFT)

Complete the function `idft()` to implement the inverse discrete Fourier transform (IDFT) equation given above. Given a frequency spectrum `X` of length `N`, the function should return its IDFT `x`, also of length `N`. Assume that the frequency index of the input spectrum ranges from 0 to `N-1`.

The input argument to the function is a numpy array `X` of the frequency spectrum and the function should return a numpy array of the IDFT of `X`.

Remember to scale the output appropriately.

```python
def idft(X):
    """
    Input:
        X (numpy array) = frequency spectrum (length N)
    Output:
        x (numpy array) = N point IDFT of the frequency spectrum X
    """
    ### Your code here

```

Now run and test the `idft()` function. If you run it with the input `X = np.array([1 ,1 ,1 ,1])`, the function should return the following numpy array:

```
array([  1.00000000e+00 +0.00000000e+00j,   -4.59242550e-17 +5.55111512e-17j,   0.00000000e+00 +6.12323400e-17j,   8.22616137e-17 +8.32667268e-17j])
```

Notice that the output numpy array is essentially `[1, 0, 0, 0]`. Instead of exact 0 we get very small numerical values of the order of 1e-15, which can be ignored. Also, these small numerical errors are machine dependent and might be different in your case.

In addition, an interesting test of the IDFT function can be done by providing the output of the DFT of a sequence as the input to the IDFT. See if you get back the original time domain sequence.

```python
# call idft
### your code here

```

## Part 5 - Compute the magnitude spectrum

Complete the function `gen_mag_spectrum()` to compute the magnitude spectrum of an input sequence `x` of length `N`. The function should return an `N` point magnitude spectrum with frequency index ranging from 0 to `N-1`.

The input argument to the function is a numpy array `x` and the function should return a numpy array of the magnitude spectrum of `x`.

```python
def gen_mag_spec(x):
    """
    Input:
        x (numpy array) = input sequence of length N
    Output:
        magX (numpy array) = magnitude spectrum of the input sequence x (length N)
    """
    ### Your code here

```

Now run and test the function `gen_mag_spec()`. If you run `gen_mag_spec()` using as input `x = np.array([1, 2, 3, 4])`, it should return the following numpy array:
```
array([10.0, 2.82842712, 2.0, 2.82842712])
```
For a more realistic use of `gen_mag_spec()` use as input a longer signal, such as `x = np.cos(2*np.pi*200.0*np.arange(512)/1000)`, and to get a visual representation of the input and output, import the `matplotlib.pyplot` package and use `plt.plot(x)` and `plt.plot(abs(X))`.

```python
import IPython.display as ipd
import matplotlib.pyplot as plt
```

```python
# call gen_mag_spec and plot result
### your code here

```


```
import numpy as np
import matplotlib.pyplot as plt
```

# Lecture 1: The Geometry of Linear Equations

$$
\begin{align}
2x -y & = 0 \\
-x +2y & = 3
\end{align}
$$
を行列で表すと
$$
\left[
  \begin{array}{cc}
    2 & -1 \\
    -1 & 2
  \end{array}
\right]
\left[
  \begin{array}{c}
    x \\
    y
  \end{array}
\right] =
\left[
  \begin{array}{r}
    0\\
    3
  \end{array}
\right]
$$

## Row Picture
行ごとに見ていく方法（方程式をそのままプロットするイメージ）

```
xs = np.linspace(-5, 5, 100)
ys1 = [2 * x for x in xs]
ys2 = [(3+x)/2 for x in xs]
fig, ax = plt.subplots()
ax.plot(xs, ys1, color='blue')
ax.plot(xs, ys2, color='red')
ax.hlines(2, -5, 5, linestyles='dotted')
ax.vlines(1, -5, 5, linestyles='dotted')
ax.annotate('[1, 2]', [1, 2])
ax.axis([-5, 5, -5, 5])
plt.show()
```

2本の線の交点が答え→$x=1, y=2$

## Column Picture
列ごとに見ていく方法（一つの列を一つのベクトルをみなす）
$$
\left[
  \begin{array}{c}
    2 \\
    -1
  \end{array}
\right]
x+
\left[
  \begin{array}{c}
    -1\\
    2
  \end{array}
\right]
y=
\left[
  \begin{array}{r}
    0\\
    3
  \end{array}
\right]
$$

```
fig, ax = plt.subplots()
ax.quiver(0, 0, 2, -1, angles='xy', scale_units='xy', scale=1, color='blue')
ax.annotate('[2, -1]', [2, -1], color='blue')
ax.quiver(0, 0, -1, 2, angles='xy', scale_units='xy', scale=1, color='red')
ax.annotate('[-1, 2]', [-1, 2], color='red')
ax.quiver(0, 0, 0, 3, angles='xy', scale_units='xy', scale=1, color='green')
ax.annotate('[0, 3]', [0, 3], color='green')
ax.axis([-4, 4, -4, 4])
plt.show()
```

青のベクトル1個分と赤のベクトル2個分を足すと緑のベクトルになる→ $x=1, y=2$


# Support Vector Machine

* Formulation

Support Vector Machine (SVM) is a supervised machine learning model that use classification  algorithm for finding decision boundary/hyperplane to separate two-group classification problem.

The hyperplane in sample space can be described using linear equation

\begin{equation}
\mathbf{w}^T \mathbf{x} + b = 0, \tag{1}
\end{equation}

here, $\mathbf{w} = (w_1, w_2, ..., w_d)$ is the normal vector of the hyperplane. Intercept $b$ denotes the distance between hyperplane and the origin point. The distance between random sample $\mathbf{x}$ and the hyperplane is

\begin{equation}
r = \frac{\left\vert \mathbf{w}^T \mathbf{x} + b \right\rvert}{\left\lVert \mathbf{w} \right\rVert} . \tag{2}
\end{equation}

* Large margin

If the hyperplane $(\mathbf{w}, b)$ is capable of properly classifying the samples. then for any samples in data space, we have

\begin{equation}
   \begin{cases}
     w_i x_i + b \geq +1, & y_i = +1\\
     w_i x_i + b \leq -1, & y_i = -1
   \end{cases}       \tag{3}
\end{equation}

Samples that satisfy $\mathbf{w}^T \mathbf{x} + b = \pm 1$ are called support vectors. Note that only support vectors are important to form the SVM model(based on KKT condition, KKT is composed of Lagrangian multiplier condition, support vectors condition and constrains of Lagrangian function).

The distance between $\mathbf{w}^T \mathbf{x} + b = +1$ and $\mathbf{w}^T \mathbf{x} + b = -1$, namely $\displaystyle \gamma = \frac{2}{\left\lVert \mathbf{w} \right\rVert}$, are called margin.

To make sure the SVM is robust and is less likely to greatly affected by small local fluctuations, we need find the large margin / maximum margin, i.e.,

\begin{equation}
\underset{(\mathbf{w}, b)}{\max} \frac{2}{\left\lVert \mathbf{w} \right\rVert},\\
s.t. y_i(\mathbf{w}^T \mathbf{x}_i + b) \geq 1, i = 1,2, ..., m. \tag{4}
\end{equation}

Here, we have $m$ samples in the data space.

This is equivalent to the following

\begin{equation}
\underset{(\mathbf{w}, b)}{\min} \frac{1}{2}\left\lVert \mathbf{w} \right\rVert^2,\\
s.t. y_i(\mathbf{w}^T \mathbf{x}_i + b) \geq 1, i = 1,2, ..., m. \tag{5}
\end{equation}

This is the base form of SVM. Obviously, it is a Convex Quadratic Programming (CQP) problem.

* Dual problem

We need to find solutions of equatin (5) to obtain the hyperplane model $f(\mathbf{x}) = \mathbf{w}^T\mathbf{x} + b$. We use Lagrangian Multiplier theorem to form the Lagrangian Function

\begin{equation}
L(\mathbf{w}, b, \mathbf{\alpha}) = \frac{1}{2}\left\lVert \mathbf{w} \right\rVert^2 - \sum\limits_{i=1}^m \alpha_i [y_i(\mathbf{w}^T \mathbf{x}_i + b) - 1] . \tag{6}
\end{equation}

Here, we assume $\mathbf{\alpha_i} \geq 0$. Set $\displaystyle \frac{\partial L(\mathbf{w}, b, \mathbf{\alpha})}{\partial \mathbf{w}} = 0$ and $\displaystyle \frac{\partial L(\mathbf{w}, b, \mathbf{\alpha})}{\partial b} = 0$. equation (6) is reorganized as

\begin{equation}
\underset{\mathbf{\alpha}}{\max} \sum\limits_{i=1}^m \alpha_i - \frac{1}{2} \sum\limits_{i=1}^m \sum\limits_{j=1}^m \alpha_i \alpha_j y_i y_j \mathbf{x}_i^T \mathbf{x}_j,\\
s.t. \sum\limits_{i=1}^{m} \alpha_i y_i = 0 \quad \text{and} \quad \alpha_i \geq 0, \quad i = 1,2, ..., m. \tag{7}
\end{equation}

Equation (7) is a dual problem of equation (5).

Once we found the multiplier $\alpha$, we can estimate $\mathbf{w}$ based on $\displaystyle \frac{\partial L(\mathbf{w}, b, \mathbf{\alpha})}{\partial \mathbf{w}} = 0$ and calculate $b$ based on large margin.

The model is then obtained

\begin{equation}
f(\mathbf{x}) = \mathbf{w}^T \mathbf{x} + b = \sum\limits_{i=1}^{m} \alpha_i y_i \mathbf{x}_i^T \mathbf{x} + b. \tag{8}
\end{equation}

* Kernel trick

SVM allows one to classify data that's linearly separable. If it isn't, one can use kernel trick to make it works.

Kernel function mapping original space to a higer dimension so that it is more easy to find a hyperplane. The hyperplane is

\begin{equation}
f(\mathbf{x}) = \mathbf{w}^T \phi(\mathbf{x}) + b . \tag{9}
\end{equation}

Here $\phi(\mathbf{x})$ is the kernel function of feature vector.

Then the SVM problem becomes

\begin{equation}
\underset{(\mathbf{w}, b)}{\min} \frac{1}{2}\left\lVert \mathbf{w} \right\rVert^2,\\
s.t. y_i(\mathbf{w}^T \phi(\mathbf{x}_i) + b) \geq 1, i = 1,2, ..., m. \tag{10}
\end{equation}

The corresponding dual problem is

\begin{equation}
\underset{\mathbf{\alpha}}{\max} \sum\limits_{i=1}^m \alpha_i - \frac{1}{2} \sum\limits_{i=1}^m \sum\limits_{j=1}^m \alpha_i \alpha_j y_i y_j \phi(\mathbf{x}_i)^T \phi(\mathbf{x}_j),\\
s.t. \sum\limits_{i=1}^{m} \alpha_i y_i = 0 \quad \text{and} \quad \alpha_i \geq 0, \quad i = 1,2, ..., m. \tag{11}
\end{equation}

It is expensive to find the inner product $\phi(\mathbf{x}_i)^T \phi(\mathbf{x}_j)$ directly, we define kernel function $K(\mathbf{x}_i, \mathbf{x}_j)$ to avoid high dimensional calculations

\begin{equation}
\underset{\mathbf{\alpha}}{\max} \sum\limits_{i=1}^m \alpha_i - \frac{1}{2} \sum\limits_{i=1}^m \sum\limits_{j=1}^m \alpha_i \alpha_j y_i y_j K(\mathbf{x}_i, \mathbf{x}_j),\\
s.t. \sum\limits_{i=1}^{m} \alpha_i y_i = 0 \quad \text{and} \quad \alpha_i \geq 0, \quad i = 1,2, ..., m. \tag{12}
\end{equation}

The model is then

\begin{equation}
f(\mathbf{x}) = \mathbf{w}^T \phi(\mathbf{x}) + b = \sum\limits_{i=1}^{m}\alpha_iy_iK(\mathbf{x}, \mathbf{x}_i) + b. \tag{13}
\end{equation}

* Soft margin

Different from the hard margin, the soft margin allows the existence of some samples that are not satisfied with the constraints $y_i(\mathbf{w}^T \mathbf{x}_i + b) \geq 1$. The optimization purpose of soft margin is to find a large margin and there are fewer samples that are not satisfied with the constraints. Therefore, the constrains of the soft margin, $y_i(\mathbf{w}^T \mathbf{x}_i + b) \geq 1-\zeta_i$, has a similar constrains but with a slack variable $\zeta_i \geq 0$ to represent overlap. Some samples are not satisfied with the constraints, so the loss function $l$ is supposed to consider in the optimization, this introduces the penalty constant $C$.

The objective of the optimization becomes

\begin{equation}
\underset{(\mathbf{w}, b)}{\min} \frac{1}{2}\left\lVert \mathbf{w} \right\rVert^2 + C \sum\limits_{i=1}^{m} l(y_i(\mathbf{w}^T \mathbf{x}_i + b) - 1),\\
s.t. y_i(\mathbf{w}^T \phi(\mathbf{x}_i) + b) \geq 1 - \zeta_i, i = 1,2, ..., m. \tag{14}
\end{equation}

The corresponding dual problem is

\begin{equation}
\underset{\mathbf{\alpha}}{\max} \sum\limits_{i=1}^m \alpha_i - \frac{1}{2} \sum\limits_{i=1}^m \sum\limits_{j=1}^m \alpha_i \alpha_j y_i y_j K(\mathbf{x}_i, \mathbf{x}_j),\\
s.t. \sum\limits_{i=1}^{m} \alpha_i y_i = 0 \quad \text{and} \quad 0 \leq \alpha_i \leq C, \quad i = 1,2, ..., m. \tag{15}
\end{equation}

* Tuning parameters

1. Penalty C: how much you want to avoid misclassifying each training sample. Large C (may lead to overfitting) results to a smaller margin even if the model did a better classification job, and vise versa. C is a penalty to the optimization of the margin so that the system pays more attention to loss function. However, as for regularization constant $\lambda$, it is a penalty to the optimization of the loss function to avoid overfitting. ($\displaystyle C=\frac{1}{\lambda}$)

2. Margin $\gamma$: how far the influence of a single training reaches. If large $\gamma$ (small standard deviation $\sigma$) is used, only samples close to the margin are considered, and vise versa. (\displaystyle $\gamma = \frac{1}{2\sigma^2}$)

* Hand-on examples

```python
pip install cvxopt
```

```python
import numpy as np

def linear_kernel(x1, x2):
    return np.dot(x1, x2)

def polynomial_kernel(x, y, c=1, degree=3):
    '''degree: great equal 1'''
    return (c + np.dot(x, y)) ** degree

def gaussian_kernel(x, y, sigma=2):
    '''sigma: great than 0'''
    gamma=1/2/(sigma ** 2)
    return np.exp(-np.linalg.norm(x-y)**2 *gamma)

def laplace_kernel(x, y, sigma):
    '''sigma: great than 0'''
    return np.exp(-np.linalg.norm(x-y) / sigma)

def sigmoid_kernel(beta, theta):
    '''beta: great than 0
       theta: less than 0'''
    return np.tanh(beta*np.dot(x, y) + theta)
```

```python
import cvxopt
import cvxopt.solvers
import matplotlib.pyplot as plt
import math as m

def train_test_split(X, Y, train_size, shuffle):
    ''' Perform tran/test datasets splitting '''
    if shuffle:
        randomize = np.arange(len(X))
        np.random.shuffle(randomize)
        X = X[randomize]
        Y = Y[randomize]
    s_id = int(len(Y) * train_size)
    X_train, X_test = X[:s_id], X[s_id:]
    Y_train, Y_test = Y[:s_id], Y[s_id:]
    return X_train, X_test, Y_train, Y_test

def generate_dataset_MVND_ls():
    '''generate a dataset that satisfies the multivariate normal distribution
       and the dataset is linearly sperable'''
    np.random.seed(24) # Fixing random state for reproducibility
    num_observations = 200
    X1 = np.random.multivariate_normal([0, 2], [[0.8, 0.6], [0.6, 0.8]], num_observations)
    X2 = np.random.multivariate_normal([2, 0], [[0.8, 0.6], [0.6, 0.8]], num_observations)
    X = np.vstack((X1, X2)).astype(np.float32)
    Y = np.hstack((np.ones(len(X1)),np.ones(len(X2))*-1))
    return X, Y

def generate_dataset_MVND_nls():
    '''generate a dataset that satisfies the multivariate normal distribution
       and the dataset is NOT linearly sperable'''
    np.random.seed(24) # Fixing random state for reproducibility
    num_observations = 100
    X1 = np.random.multivariate_normal([-1, 2], [[1.0,0.8], [0.8, 1.0]], num_observations)
    X1 = np.vstack((X1, np.random.multivariate_normal([4, -4], [[1.0,0.8], [0.8, 1.0]], num_observations)))
    X2 = np.random.multivariate_normal([1, -1], [[1.0,0.8], [0.8, 1.0]], num_observations)
    X2 = np.vstack((X2, np.random.multivariate_normal([-4, 4], [[1.0,0.8], [0.8, 1.0]], num_observations)))
    X = np.vstack((X1, X2)).astype(np.float32)
    Y = np.hstack((np.ones(len(X1)),np.ones(len(X2))*-1))
    return X, Y

def generate_dataset_MVND_lso():
    '''generate a dataset that satisfies the multivariate normal distribution
       and the dataset is linearly sperable but overlapping'''
    np.random.seed(24) # Fixing random state for reproducibility
    num_observations = 100
    X1 = np.random.multivariate_normal([0, 2], [[1.5, 1.0], [1.0, 1.5]], num_observations)
    X2 = np.random.multivariate_normal([2, 0], [[1.5, 1.0], [1.0, 1.5]], num_observations)
    X = np.vstack((X1, X2)).astype(np.float32)
    Y = np.hstack((np.ones(len(X1)),np.ones(len(X2))*-1))
    return X, Y

def generate_dataset_MVND_nlso2():
    '''generate a dataset that satisfies the multivariate normal distribution
       and the dataset is NOT linearly sperable but overlapping'''
    np.random.seed(24) # Fixing random state for reproducibility
    num_observations = 100
    radius1 = np.sqrt(np.array(np.random.uniform(0,1,num_observations))).reshape(-1,1)
    angle1 = 2*m.pi*np.array(np.random.uniform(0,1,num_observations)).reshape(-1,1)
    X1 = np.array(radius1*np.cos(angle1)).reshape(-1,1)
    X2 = np.array(radius1*np.sin(angle1)).reshape(-1,1)
    X_ = np.hstack((X1, X2))
    radius2 = np.sqrt(np.array(3.* np.random.uniform(0,1,num_observations) + .9)).reshape(-1,1)
    X3 = np.array(radius2*np.cos(angle1)).reshape(-1,1)
    X4 = np.array(radius2*np.sin(angle1)).reshape(-1,1)
    X_X = np.hstack((X3, X4))
    X = np.vstack((X_, X_X))
    Y = np.hstack((np.ones(len(X_)),np.ones(len(X_X))*-1))
    return X, Y

def generate_dataset_MVND_nls2():
    '''generate a dataset that satisfies the multivariate normal distribution
       and the dataset is NOT linearly sperable'''
    np.random.seed(24) # Fixing random state for reproducibility
    num_observations = 100
    radius1 = np.sqrt(np.array(np.random.uniform(0,1,num_observations))).reshape(-1,1)
    angle1 = 2*m.pi*np.array(np.random.uniform(0,1,num_observations)).reshape(-1,1)
    X1 = np.array(radius1*np.cos(angle1)).reshape(-1,1)
    X2 = np.array(radius1*np.sin(angle1)).reshape(-1,1)
    X_ = np.hstack((X1, X2))
    radius2 = np.sqrt(np.array(3.* np.random.uniform(0,1,num_observations) + 1.5)).reshape(-1,1)
    X3 = np.array(radius2*np.cos(angle1)).reshape(-1,1)
    X4 = np.array(radius2*np.sin(angle1)).reshape(-1,1)
    X_X = np.hstack((X3, X4))
    X = np.vstack((X_, X_X))
    Y = np.hstack((np.ones(len(X_)),np.ones(len(X_X))*-1))
    return X, Y

def metric_accuracy_count(Y_label, Y_pred):
    '''Evaluate the accuracy'''
    correct_amount = 0
    for i in range(np.size(Y_pred)) :   #np.size: Number of elements in the array
        if Y_label[i] == Y_pred[i] :
            correct_amount = correct_amount + 1
    return correct_amount

def plot_contour(X1_train, X2_train, model):
    plt.figure(figsize = (12, 8))
    plt.scatter(X1_train[:,0], X1_train[:,1], s=50, c="r", cmap=plt.cm.jet, marker = '+', label = 'Positive ')
    plt.scatter(X2_train[:,0], X2_train[:,1], s=50, c="b", cmap=plt.cm.jet, marker = '_', label = 'Negative ')
    plt.scatter(clf.support_vector[:,0], clf.support_vector[:,1], s=120, facecolors='none', edgecolors='g', label = 'Support Vector')

    X1, X2 = np.meshgrid(np.linspace(-6,6,50), np.linspace(-6,6,50))
    X = np.array([[x1, x2] for x1, x2 in zip(np.ravel(X1), np.ravel(X2))])
    Z = clf.predict(X).reshape(X1.shape)
    plt.contour(X1, X2, Z, [0.0], colors='k', linewidths=1, origin='lower')
    plt.contour(X1, X2, Z + 1, [0.0], colors='grey', linestyles = 'dashed', linewidths=1, origin='lower')
    plt.contour(X1, X2, Z - 1, [0.0], colors='grey', linestyles = 'dashed', linewidths=1, origin='lower')

    plt.show()

class SVM():
    '''Suppor Vector Machine model.
       Used to find the decision boundary/hyperplane
       to separate the two-group classification problem
       ------------------------------------------------
       kernel: kernel function of kernel trick,
               designed for cases that are not linearly separable
       C: penality constant (inverse of regularization constant, 1/lambda).
               It means how much you want to avoid misclassifying each sample.
               If large C is used, then the model tries to make sure all samples
               satisfy the constrains, namely, a smaller-margin hyperplane is
               obtained even if the samples are all correctly classified, and
               vise versa.'''

    def __init__(self, kernel, C=None):
        self.kernel = kernel
        self.C = C

    def fit(self, X, y):
        # m instances, d atrributes
        self.m, self.d = X.shape
        # Kernel matrix
        K = np.zeros((self.m, self.m))
        for i in range(self.m):
            for j in range(self.m):
                K[i,j] = self.kernel(X[i], X[j])
        # The Convex Quadratic Programming parameters (https://cvxopt.org/examples/tutorial/qp.html)
        # QP Problem like: min_x 0.5*x^T P x + q^T x, s.t. G x <= h and A x = b.
        # May use list or np.array to create matrix. Note that if use list,
        # matrixs should be defined by columns instead of rows
        P = cvxopt.matrix(np.outer(y,y) * K)
        q = cvxopt.matrix(np.ones(self.m) * -1)
        A = cvxopt.matrix(y, (1, self.m))
        b = cvxopt.matrix(0.0)
        if not self.C:
            G = cvxopt.matrix(np.diag(np.ones(self.m) * -1))
            h = cvxopt.matrix(np.zeros(self.m))
        else:
            G1 = np.diag(np.ones(self.m) * -1)
            G2 = np.identity(self.m)
            G = cvxopt.matrix(np.vstack((G1, G2)))
            h1 = np.zeros(self.m)
            h2 = np.ones(self.m) * self.C
            h = cvxopt.matrix(np.hstack((h1, h2)))

        # Construct the QP
        sol = cvxopt.solvers.qp(P, q, G, h, A, b)
        # Lagrange multipliers
        l_m = np.ravel(sol['x'])

        # The model, in its formation, is:　f(x) = w^T x + b = (sum_i^m l_m_i y_i K（x_i^T, x) + b
        # if l_m_i = 0, it will not exist in the above summation, and will have no influence on f(x),
        # only consider cases that have l_m_i > 0. Based on KKT condition, if l_m_i > 0, y_if(x_i) = 1.
        # That means these samples are right on the large margins and they are support vectors.
        # Only the support vectors are important in training the model f(x).
        support_vector_id = l_m > 1e-6
        new_id = np.arange(len(l_m))[support_vector_id]
        store_id = [i for i in range(len(l_m)) if support_vector_id[i]==True]
        store_id = np.array(store_id)
        self.l_m = l_m[support_vector_id]
        self.support_vector = X[support_vector_id]
        self.support_vector_label = y[support_vector_id]

        # Estimate the hyperplane intercept
        self.b = 0
        for i in range(len(self.l_m)):
            self.b += self.support_vector_label[i] / len(self.l_m)
            self.b -= np.sum(self.l_m * self.support_vector_label * K[store_id[i],support_vector_id]) / len(self.l_m)

    def predict(self, X):
        self.y_pred = np.zeros(len(X))
        for i in range(len(X)):
            tmp = 0
            for j in range(len(self.l_m)):
                tmp = tmp + self.l_m[j] * self.support_vector_label[j] * self.kernel(X[i], self.support_vector[j])
            self.y_pred[i] = tmp + self.b
        return self.y_pred

```

```python
def main():

    # Import data
    X, Y = generate_dataset_MVND_ls()
    print(X.shape, Y.shape)
    # Splitting dataset into train and test set
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=.9, shuffle=False)
    print( X_train.shape, Y_train.shape)
    print( X_test.shape, Y_test.shape)
    # Model Learning
    model = SVM(linear_kernel)
    model.fit(X_train, Y_train)
    # Model Working
    Y_pred = model.predict(X_test)
    Y_pred = np.sign(Y_pred)
    #Statistics
    #print( 'Accuracy count: ', metric_accuracy_count(Y_test, Y_pred), ' out of ', len(Y_test), ' are correct!' )
    #Visulaization
    plt.figure(figsize = (12, 8))
    c1 = np.ma.masked_where(Y == -1, Y)
    c2 = np.ma.masked_where(Y == 1, Y)
    plt.scatter(X[:,0], X[:,1], s=50, c=c1, cmap="RdBu_r", marker = '+', label = 'Positive ')
    plt.scatter(X[:,0], X[:,1], s=50, c=c2, cmap="RdBu_r", marker = '_', label = 'Negative ')
    plt.scatter(model.support_vector[:,0], model.support_vector[:,1], s=120, facecolors='none', edgecolors='c', label = 'Support Vectors')
    X1, X2 = np.meshgrid(np.linspace(-3,5,10), np.linspace(-3,5,10))
    X = np.array([[x1, x2] for x1, x2 in zip(np.ravel(X1), np.ravel(X2))])
    Y = model.predict(X).reshape(X1.shape)
    plt.contour(X1, X2, Y, [0], colors='k', linewidths=3) # W^t X = 0
    plt.contour(X1, X2, Y - 1, [0], colors='k', linestyles = 'dashed', linewidths=1) # W^t X = 1
    plt.contour(X1, X2, Y + 1, [0], colors='k', linestyles = 'dashed', linewidths=1) # W^t X = -1

    plt.legend()
    plt.show()

if __name__ == "__main__":
    main()
```

```python
def main():

    # Import data
    X, Y = generate_dataset_MVND_lso()
    print(X.shape, Y.shape)
    # Splitting dataset into train and test set
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=.9, shuffle=False)
    print( X_train.shape, Y_train.shape)
    print( X_test.shape, Y_test.shape)
    # Model Learning
    model = SVM(linear_kernel, C=1000)
    model.fit(X_train, Y_train)
    # Model Working
    Y_pred = model.predict(X_test)
    Y_pred = np.sign(Y_pred)
    #Statistics
    #print( 'Accuracy count: ', metric_accuracy_count(Y_test, Y_pred), ' out of ', len(Y_test), ' are correct!' )
    #Visulaization
    plt.figure(figsize = (12, 8))
    c1 = np.ma.masked_where(Y == -1, Y)
    c2 = np.ma.masked_where(Y == 1, Y)
    plt.scatter(X[:,0], X[:,1], s=50, c=c1, cmap="RdBu_r", marker = '+', label = 'Positive ')
    plt.scatter(X[:,0], X[:,1], s=50, c=c2, cmap="RdBu_r", marker = '_', label = 'Negative ')
    plt.scatter(model.support_vector[:,0], model.support_vector[:,1], s=120, facecolors='none', edgecolors='c', label = 'Support Vectors')
    X1, X2 = np.meshgrid(np.linspace(-3,5,10), np.linspace(-3,5,10))
    X = np.array([[x1, x2] for x1, x2 in zip(np.ravel(X1), np.ravel(X2))])
    Y = model.predict(X).reshape(X1.shape)
    plt.contour(X1, X2, Y, [0], colors='k', linewidths=3) # W^t X = 0
    plt.contour(X1, X2, Y - 1, [0], colors='k', linestyles = 'dashed', linewidths=1) # W^t X = 1
    plt.contour(X1, X2, Y + 1, [0], colors='k', linestyles = 'dashed', linewidths=1) # W^t X = -1

    plt.legend()
    plt.show()

if __name__ == "__main__":
    main()

```

```python
def main():

    # Import data
    X, Y = generate_dataset_MVND_nls()
    print(X.shape, Y.shape)
    # Splitting dataset into train and test set
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=.9, shuffle=False)
    print( X_train.shape, Y_train.shape)
    print( X_test.shape, Y_test.shape)
    # Model Learning
    model = SVM(polynomial_kernel)
    model.fit(X_train, Y_train)
    # Model Working
    Y_pred = model.predict(X_test)
    Y_pred = np.sign(Y_pred)
    #Statistics
    #print( 'Accuracy count: ', metric_accuracy_count(Y_test, Y_pred), ' out of ', len(Y_test), ' are correct!' )
    #Visulaization
    plt.figure(figsize = (12, 8))
    c1 = np.ma.masked_where(Y == -1, Y)
    c2 = np.ma.masked_where(Y == 1, Y)
    plt.scatter(X[:,0], X[:,1], s=50, c=c1, cmap="RdBu_r", marker = '+', label = 'Positive ')
    plt.scatter(X[:,0], X[:,1], s=50, c=c2, cmap="RdBu_r", marker = '_', label = 'Negative ')
    plt.scatter(model.support_vector[:,0], model.support_vector[:,1], s=120, facecolors='none', edgecolors='c', label = 'Support Vectors')
    X1, X2 = np.meshgrid(np.linspace(-7,7,100), np.linspace(-7,7,100))
    X = np.array([[x1, x2] for x1, x2 in zip(np.ravel(X1), np.ravel(X2))])
    Y = model.predict(X).reshape(X1.shape)
    plt.contour(X1, X2, Y, [0], colors='k', linewidths=3) # W^t X = 0
    plt.contour(X1, X2, Y - 1, [0], colors='k', linestyles = 'dashed', linewidths=1) # W^t X = 1
    plt.contour(X1, X2, Y + 1, [0], colors='k', linestyles = 'dashed', linewidths=1) # W^t X = -1

    plt.legend()
    plt.show()

if __name__ == "__main__":
    main()
```

```python
def main():

    # Import data
    X, Y = generate_dataset_MVND_nls2()
    print(X.shape, Y.shape)
    # Splitting dataset into train and test set
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=.9, shuffle=False)
    print( X_train.shape, Y_train.shape)
    print( X_test.shape, Y_test.shape)
    # Model Learning
    model = SVM(gaussian_kernel)
    model.fit(X_train, Y_train)
    # Model Working
    Y_pred = model.predict(X_test)
    Y_pred = np.sign(Y_pred)
    #Statistics
    #print( 'Accuracy count: ', metric_accuracy_count(Y_test, Y_pred), ' out of ', len(Y_test), ' are correct!' )
    #Visulaization
    plt.figure(figsize = (8, 8))
    c1 = np.ma.masked_where(Y == -1, Y)
    c2 = np.ma.masked_where(Y == 1, Y)
    plt.scatter(X[:,0], X[:,1], s=50, c=c1, cmap="RdBu_r", marker = '+', label = 'Positive ')
    plt.scatter(X[:,0], X[:,1], s=50, c=c2, cmap="RdBu_r", marker = '_', label = 'Negative ')
    plt.scatter(model.support_vector[:,0], model.support_vector[:,1], s=120, facecolors='none', edgecolors='c', label = 'Support Vectors')
    X1, X2 = np.meshgrid(np.linspace(-3,3,100), np.linspace(-3,3,100))
    X = np.array([[x1, x2] for x1, x2 in zip(np.ravel(X1), np.ravel(X2))])
    Y = model.predict(X).reshape(X1.shape)
    plt.contour(X1, X2, Y, [0], colors='k', linewidths=3) # W^t X = 0
    plt.contour(X1, X2, Y - 1, [0], colors='k', linestyles = 'dashed', linewidths=1) # W^t X = 1
    plt.contour(X1, X2, Y + 1, [0], colors='k', linestyles = 'dashed', linewidths=1) # W^t X = -1

    plt.legend()
    plt.show()

if __name__ == "__main__":
    main()
```

```python

```

```python

```

```python

```

Support Vector Machine/Support Vector Machine.ipynb


# Understanding the FFT Algorithm

https://jakevdp.github.io/blog/2013/08/28/understanding-the-fft/

The Fast Fourier Transform (FFT) is one of the most important algorithms in signal processing and data analysis. I've used it for years, but having no formal computer science background, It occurred to me this week that I've never thought to ask how the FFT computes the discrete Fourier transform so quickly. I dusted off an old algorithms book and looked into it, and enjoyed reading about the deceptively simple computational trick that JW Cooley and John Tukey outlined in their classic 1965 paper introducing the subject.

The goal of this post is to dive into the Cooley-Tukey FFT algorithm, explaining the symmetries that lead to it, and to show some straightforward Python implementations putting the theory into practice. My hope is that this exploration will give data scientists like myself a more complete picture of what's going on in the background of the algorithms we use.

For simplicity, we'll concern ourself only with the forward transform, as the inverse transform can be implemented in a very similar manner. Taking a look at the DFT expression above, we see that it is nothing more than a straightforward linear operation: a matrix-vector multiplication of$\vec{x}$

Forward Discrete Fourier Transform (DFT):

$$X_k = \sum_{n=0}^{N-1} x_n \cdot e^{-i~2\pi~k~n~/~N}$$

Inverse Discrete Fourier Transform (IDFT):

$$x_n = \frac{1}{N}\sum_{k=0}^{N-1} X_k e^{i~2\pi~k~n~/~N}$$

```python
import numpy as np
def DFT_slow(x):
    """Compute the discrete Fourier Transform of the 1D array x"""
    x = np.asarray(x, dtype=float)
    N = x.shape[0]
    n = np.arange(N)
    k = n.reshape((N, 1))
    M = np.exp(-2j * np.pi * k * n / N)
    return np.dot(M, x)
#We can double-check the result by comparing to numpy's built-in FFT function:
x = np.random.random(1024)
np.allclose(DFT_slow(x), np.fft.fft(x))
```

    True

Just to confirm the sluggishness of our algorithm, we can compare the execution times of these two approaches:

```python
%timeit DFT_slow(x)
%timeit np.fft.fft(x)
```

    111 ms ± 19.9 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
    3.89 µs ± 59.5 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)

We are over 1000 times slower, which is to be expected for such a simplistic implementation. But that's not the worst of it. For an input vector of length N, the FFT algorithm scales as$ O[NlogN]$, while our slow algorithm scales as $O[N2]$. That means that for $N=10^6$ elements, we'd expect the FFT to complete in somewhere around 50 ms, while our slow algorithm would take nearly 20 hours!

So how does the FFT accomplish this speedup? The answer lies in exploiting symmetry.

# Symmetries in the Discrete Fourier Transform

One of the most important tools in the belt of an algorithm-builder is to exploit symmetries of a problem. If you can show analytically that one piece of a problem is simply related to another, you can compute the subresult only once and save that computational cost. Cooley and Tukey used exactly this approach in deriving the FFT.

We'll start by asking what the value of $X_{N+k}$ is. From our above expression:

$$\begin{align*}
X_{N + k} &=  \sum_{n=0}^{N-1} x_n \cdot e^{-i~2\pi~(N + k)~n~/~N}\\
          &= \sum_{n=0}^{N-1} x_n \cdot e^{- i~2\pi~n} \cdot e^{-i~2\pi~k~n~/~N}\\
          &= \sum_{n=0}^{N-1} x_n \cdot e^{-i~2\pi~k~n~/~N}
\end{align*}$$

where we've used the identity $exp[2π i n]=1$ which holds for any integer n.

The last line shows a nice symmetry property of the DFT:

$$X_{N+k}=X_k$$

By a simple extension,

$$X_{k + i \cdot N} = X_k$$

for any integer i. As we'll see below, this symmetry can be exploited to compute the DFT much more quickly.

# DFT to FFT: Exploiting Symmetry

Cooley and Tukey showed that it's possible to divide the DFT computation into two smaller parts. From the definition of the DFT we have:

$$\begin{align}
X_k &= \sum_{n=0}^{N-1} x_n \cdot e^{-i~2\pi~k~n~/~N} \\
    &= \sum_{m=0}^{N/2 - 1} x_{2m} \cdot e^{-i~2\pi~k~(2m)~/~N} + \sum_{m=0}^{N/2 - 1} x_{2m + 1} \cdot e^{-i~2\pi~k~(2m + 1)~/~N} \\
    &= \sum_{m=0}^{N/2 - 1} x_{2m} \cdot e^{-i~2\pi~k~m~/~(N/2)} + e^{-i~2\pi~k~/~N} \sum_{m=0}^{N/2 - 1} x_{2m + 1} \cdot e^{-i~2\pi~k~m~/~(N/2)}
\end{align}$$

We've split the single Discrete Fourier transform into two terms which themselves look very similar to smaller Discrete Fourier Transforms, one on the odd-numbered values, and one on the even-numbered values. So far, however, we haven't saved any computational cycles. Each term consists of $(N/2)∗N $computations, for a total of $N^2$.

The trick comes in making use of symmetries in each of these terms. Because the range of $k$ is $0≤k<N$, while the range of$ n$ is $0≤n<M≡N/2$, we see from the symmetry properties above that we need only perform half the computations for each sub-problem. Our$ O[N^2]$ computation has become $O[M^2]$, with M half the size of $N$.

But there's no reason to stop there: as long as our smaller Fourier transforms have an even-valued M, we can reapply this divide-and-conquer approach, halving the computational cost each time, until our arrays are small enough that the strategy is no longer beneficial. In the asymptotic limit, this recursive approach scales as $O[NlogN]$.

This recursive algorithm can be implemented very quickly in Python, falling-back on our slow DFT code when the size of the sub-problem becomes suitably small:

```python
import numpy as np
def FFT(x):
    """A recursive implementation of the 1D Cooley-Tukey FFT"""
    x = np.asarray(x, dtype=float)
    N = x.shape[0]

    if N % 2 > 0:
        raise ValueError("size of x must be a power of 2")
    elif N <= 32:  # this cutoff should be optimized
        return DFT_slow(x)
    else:
        X_even = FFT(x[::2])
        X_odd = FFT(x[1::2])
        factor = np.exp(-2j * np.pi * np.arange(N) / N)
        return np.concatenate([X_even + factor[:N / 2] * X_odd,
                               X_even + factor[N / 2:] * X_odd])
```

```python
x = np.random.random(1024)
np.allclose(FFT(x), np.fft.fft(x))
```

```python

```


# Automatic differentiation

__Automatic differentiation__ is a method for evaluating the rate of change in the numerical output of a program with respect to the rate of change in its input. The power of the method is the ability of writing a program that computes a differentiable function and having the derivative immediatelly available.

We start with an example.

Consider the function
$$
f(x) = \cos(x)\sin(x)
$$

When we want to evaluate the function numerically at a specific $x$, say $x=1$ we can implement a computer program like
~~~
x = 1
f = cos(x)*sin(x)
~~~

or

~~~
def g(x):
    return cos(x)*sin(x)

x = 1
f = g(x)
~~~

Now suppose we need the derivative as well, that is how much $f$ changes when we slightly change $x$. For this example, it is a simple exercise to calculate the derivative symbolically as
$$
f'(x) = \cos(x)\cos(x) -\sin(x)\sin(x) = \cos(x)^2 - \sin(x)^2
$$
and code this explicitely as
~~~
x = 1
df_dx =  cos(x)*cos(x) - sin(x)*sin(x)
~~~

But could we have calculated the derivative without coding it up explicitely, that is without symbolically evaluating it a priori by hand? For example, can we code just
~~~
x = my.Variable(1)
f = my.cos(x)*my.sin(x)
df_dx = f.derivative()
~~~
or

~~~
def g(x):
    return my.cos(x)*my.sin(x)

x = my.Variable(1)
f = g(x)
df_dx = f.derivative()
~~~

to get what we want, perhaps by overloading the appropriate variables, functions and operators? The answer turns out to be yes and it is a quite fascinating subject called __automatic differentiation__. Interestingly, this algorithm, known also as __backpropagation__, is in the core of todays artificial intelligence systems, programs that learn how to program themselves from input and output examples. See https://www.youtube.com/watch?v=aircAruvnKk for an introduction to a particular type of model, known as a __neural network__.

To symbolically evaluate the derivative, we use the chain rule. The chain rule dictates that when
$$
f(x) = g(h(x))
$$
the derivative is given as
$$
f'(x) = g'(h(x)) h'(x)
$$

We could implement this program as
~~~
x = 1
h = H(x)
g = G(h)
f = g
~~~
where we have used capital letters for the functions -- beware that the function and its output is always denoted with the same letter in mathematical notation. To highlight the underlying mechanism of automatic differentiation, we will always assign the output of a function to a variable so we will only think of the rate of change of a variable with respect to another variable, rather than 'derivatives of functions'. To be entirely formal we write
~~~
x = 1
h = H(x)
g = G(h)
f = identity(g)
~~~
and denote the identity function as $(\cdot)$. This program can be represented also by the following directed computation graph:

The derivative is denoted by
$$
f'(x)  =  \frac{df}{dx}
$$
As we will later use multiple variables, we will already introduce the partial derivative notation, that is equivalent to the derivative for scalar functions.
$$
f'(x)  =  \frac{\partial f}{\partial x}
$$

The chain rule, using the partial derivative notation can be stated as
\begin{eqnarray}
\frac{\partial f}{\partial x} & = &  \frac{\partial h}{\partial x} \frac{\partial g}{\partial h} \frac{\partial f}{\partial g} \\
& = & h'(x) g'(h(x)) \cdot 1
\end{eqnarray}

This quantity is actually just a product of numbers, so we could have evaluated this derivative in the following order
\begin{eqnarray}
\frac{\partial f}{\partial x} & = & \frac{\partial h}{\partial x} \left(\frac{\partial g}{\partial h} \left(\frac{\partial f}{\partial g} \frac{\partial f}{\partial f} \right) \right) \\
& =& \frac{\partial h}{\partial x} \left(\frac{\partial g}{\partial h} \frac{\partial f}{\partial g} \right) \\
& = &\frac{\partial h}{\partial x} \frac{\partial f}{\partial h}  \\
& = &\frac{\partial f}{\partial x}
\end{eqnarray}
where we have included $\partial f/\partial f = 1$ as the boundry case.

So, we could imagine calculating the derivative using the following program

~~~
df_df = 1
df_dg = 1 * df_df
df_dh = dG(h) * df_dg
df_dx = dH(x) * df_dh
~~~

If $g$ and $h$ are elementary functions, their derivatives are known in closed form and can be calculated from their input(s) only.

As an example, consider
$$
f(x) = \sin(\cos(x))
$$

The derivative is
$$
\frac{\partial f}{\partial x}  =  -\sin(x) \cos(\cos(x))
$$

~~~
df_df =  1
df_dg =  1 * df_df
df_dh =  cos(h) * df_dg
df_dx = -sin(x) * df_dh
~~~

As $h=\cos(x)$, it can be easily verified that the derivative is calculated correctly.

### Functions of two or more variables
When we have functions of two or more variables the notion of a derivative changes slightly. For example, when
$$
g(x_1, x_2)
$$
we define the partial derivatives
\begin{eqnarray}
\frac{\partial g}{\partial x_1} & , &  \frac{\partial g}{\partial x_2}
\end{eqnarray}

The collection of partial derivatives can be organized as a vector. This object is known as the __gradient__ and is denoted as
\begin{eqnarray}
\nabla g(x) \equiv \left(\begin{array}{c} \frac{\partial g}{\partial x_1} \\ \frac{\partial g}{\partial x_2} \end{array} \right)
\end{eqnarray}

When taking the partial derivative, we assume that all the variables are constant, apart from the one that we are taking the derivative with respect to.

For example,
$$
g(x_1, x_2) = \cos(x_1)e^{3 x_2}
$$
When taking the (partial) derivative with respect to $x_1$, we assume that the second factor is a constant
$$
\frac{\partial g}{\partial x_1} = -\sin(x_1) e^{3 x_2}
$$
Similarly, when taking the partial derivative with respect to $x_2$, we assume that the first factor is a constant
$$
\frac{\partial g}{\partial x_2} = 3 \cos(x_1) e^{3 x_2}
$$

The chain rule for multiple variables is in a way similar to the chain rule for single variable functions but with a caveat: the derivatives over all paths between the two variables need to be added.

Another example is
$$
f(x) = g(h_1(x), h_2(x))
$$
Here, the partial derivative is
$$
\frac{\partial g}{\partial x} = \frac{\partial g}{\partial h_1} \frac{\partial h_1}{\partial x} +   \frac{\partial g}{\partial h_2} \frac{\partial h_2}{\partial x}
$$
The chain rule has a simple form
$$
\frac{\partial f}{\partial x}  = \frac{\partial f}{\partial g} \frac{\partial g}{\partial x}
$$

To see a concrete example of a function of form $f(x) = g(h_1(x), h_2(x)) $, consider
$$
f(x) = \sin(x)\cos(x)
$$

We define
\begin{align}
h_1(x) & = c = \cos(x) \\
h_2(x) & = s = \sin(x) \\
g(c,s) & = g = c \times s \\
f & = g(c,s)
\end{align}

that is equivalent to the following program, written deliberately as a sequence of scalar function evaluations and binary operators only
~~~
x = 1
c = cos(x)
s = sin(x)
g = c * s
f = g
~~~

This program can be represented by the following directed computation graph:

The function can be evaluated by traversing the variable nodes of the directed graph from the inputs to the outputs in the topological order. At each variable node, we merely evaluate the incoming function. Topological order guarantees that the inputs for the function are already calculated.

It is not obvious, but the derivatives can also be calculated easily. By the chain rule, we have
\begin{eqnarray}
\frac{\partial f}{\partial x} &=& \frac{\partial f}{\partial g} \frac{\partial g}{\partial c} \frac{\partial c}{\partial x} +  \frac{\partial f}{\partial g} \frac{\partial g}{\partial s} \frac{\partial s}{\partial x} \\
&=& 1 \cdot s \cdot (-\sin(x)) +  1 \cdot c \cdot \cos(x) \\
&=& -\sin(x) \cdot \sin(x) +  1 \cdot \cos(x) \cdot \cos(x) \\
\end{eqnarray}

The derivative could have been calculated numerically by the following program

~~~
df_dx = 0, df_ds = 0, df_dc = 0, df_dg = 0
df_df = 1

df_dg +=  df_df           // df/dg = 1
df_dc +=  s * df_dg       // dg/dc = s
df_ds +=  c * df_dg       // dg/ds = c
df_dx +=  cos(x) * df_ds  // ds/dx = cos(x)
df_dx += -sin(x) * df_dc  // dc/dx = -sin(x)
~~~

Note that the total derivative consists of sums of several terms. Each term is the product of the derivatives along the path leading from $f$ to $x$. In the above example, there are only two paths:

- $f,g,c,x$
- $f,g,s,x$

$$
\frac{\partial f}{\partial x}  = \frac{\partial f}{\partial g} \frac{\partial g}{\partial c} \frac{\partial c}{\partial x} +  \frac{\partial f}{\partial g} \frac{\partial g}{\partial s} \frac{\partial s}{\partial x}
$$

It is not obvious in this simple example but the fact that we are propagating backwards makes us save computation by storing the intermediate variables.

This program can be represented by the following directed computation graph:

Note that during the backward pass, if we traverse variable nodes in the reverse topological order, we only need the derivatives already computed in previous steps and values of variables that are connected to the function node that are computed during the forward pass. As an example, consider

$$
\frac{\partial f}{\partial c} = \frac{\partial f}{\partial g} \frac{\partial g}{\partial c}
$$
The first term is already available during the backward pass. The second term needs to be programmed by calculating the partial derivative of $g(s,c) = sc$ with respect to $c$. It has a simple form, namely $s$. More importantly, the numerical value is also immediately available, as it is calculated during the forward pass. For each function type, this calculation will be different but is nevertheless straightforward for all basic functions, including the binary arithmetic operators $+,-,\times$ and $\div$.

Tutorial introductions to Automatic differentiation

Richard D. Neidinger,
Introduction to Automatic Differentiation and MATLAB Object-Oriented Programming,
SIAM REVIEW, 2010 Society for Industrial and Applied Mathematics,
Vol. 52, No. 3, pp. 545–563

Baydin, Atılım Güneş, Barak A. Pearlmutter, Alexey Andreyevich Radul, and Jeffrey Mark Siskind. 2018. “Automatic Differentiation in Machine Learning: a Survey.” Journal of Machine Learning Research (JMLR)
https://arxiv.org/pdf/1502.05767.pdf

Two related blog posts from Ben Recht:

http://www.argmin.net/2016/05/18/mates-of-costate/

http://www.argmin.net/2016/05/31/mechanics-of-lagrangians/

Back-propagation, an introduction, by Sanjeev Arora and Tengyu Ma
http://www.offconvex.org/2016/12/20/backprop/

A nice autodifferentiation package for python
https://github.com/HIPS/autograd

A good tutorial on Backpropagation by Roger Grosse

http://www.cs.toronto.edu/~rgrosse/courses/csc321_2017/slides/lec6.pdf

http://www.cs.toronto.edu/~rgrosse/courses/csc321_2017/readings/L06%20Backpropagation.pdf

```python
from __future__ import absolute_import
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad

'''
Mathematically we can only take gradients of scalar-valued functions, but
autograd's grad function also handles numpy's familiar vectorization of scalar
functions, which is used in this example.
To be precise, grad(fun)(x) always returns the value of a vector-Jacobian
product, where the Jacobian of fun is evaluated at x and the the vector is an
all-ones vector with the same size as the output of fun. When vectorizing a
scalar-valued function over many arguments, the Jacobian of the overall
vector-to-vector mapping is diagonal, and so this vector-Jacobian product simply
returns the diagonal elements of the Jacobian, which is the gradient of the
function at each input value over which the function is vectorized.
'''

def tanh(x):
    return (1.0 - np.exp(-x))  / (1.0 + np.exp(-x))

x = np.linspace(-7, 7, 200)
plt.plot(x, tanh(x),
         x, grad(tanh)(x),                                 # first derivative
         x, grad(grad(tanh))(x),                           # second derivative
         x, grad(grad(grad(tanh)))(x),                     # third derivative
         x, grad(grad(grad(grad(tanh))))(x),               # fourth derivative
         x, grad(grad(grad(grad(grad(tanh)))))(x),         # fifth derivative
         x, grad(grad(grad(grad(grad(grad(tanh))))))(x))   # sixth derivative

plt.axis('off')
plt.savefig("tanh.png")
plt.show()
```