```python
# import Python libraries
import numpy as np
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
import sympy as sym
from sympy.plotting import plot
import pandas as pd
from IPython.display import display
from IPython.core.display import Math
from scipy.optimize import minimize
```

1. Find the extrema in the function $f(x)=x^3-7.5x^2+18x-10$ analytically and determine if they are minimum or maximum.

```python
x = sym.symbols('x')
f = x**3 - 7.5*x**2 + 18*x - 10
display(Math(sym.latex('f(x) = ') + sym.latex(f)))

fdiff = sym.expand(sym.diff(f, x))
display(Math(sym.latex('\dot f(x) = ') + sym.latex(fdiff)))

roots = sym.solve(fdiff, x)
display(Math(sym.latex('Roots:') + sym.latex(roots)))

fdiff2 = sym.expand(sym.diff(fdiff, x))
display(Math(sym.latex('\ddot f(x) = ') + sym.latex(fdiff2)))

f2 = fdiff2.subs(x,2)
display(Math(sym.latex('\ddot f(2) = ') + sym.latex(f2)))

f3 = fdiff2.subs(x,3)
display(Math(sym.latex('\ddot f(3) = ') + sym.latex(f3)))
```

$$f(x) = x^{3} - 7.5 x^{2} + 18 x - 10$$

$$\dot f(x) = 3 x^{2} - 15.0 x + 18$$

$$Roots:\left [ 2.0, \quad 3.0\right ]$$

$$\ddot f(x) = 6 x - 15.0$$

$$\ddot f(2) = -3.0$$

$$\ddot f(3) = 3.0$$

f(2) is a maximum and f(3) is a minimum

```python
plot(f,(x,1,4),xlabel= 'x',ylabel = 'f(x)')
```

```python

```

2. Find the minimum in the $f(x)=x^3-7.5x^2+18x-10$ using the gradient descent algorithm.

```python
cur_x = 2.001
gamma = 0.01            # step size multiplier
precision = 0.00001
step_size = 1           # initial step size
max_iters = 10000       # maximum number of iterations
iters = 0               # iteration counter

f  = lambda x: x**3 - 7.5*x**2 + 18*x - 10  # lambda function for f(x)
df = lambda x: 3*x**2 - 15*x + 18   # lambda function for the gradient of f(x)

while (step_size > precision) & (iters < max_iters):
    prev_x = cur_x
    cur_x -= gamma*df(prev_x)
    step_size = abs(cur_x - prev_x)
    iters+=1

print('True local minimum at {} with function value {}.'.format(3, f(3)))
print('Local minimum by gradient descent at {} with function value {}.'.format(cur_x, f(cur_x)))
```

    True local minimum at 3 with function value 3.5.
    Local minimum by gradient descent at 2.9996813387653187 with function value 3.500000152285125.

```python

```

3. Regarding the distribution problem for the elbow muscles presented in this text:
    a. Test different initial values for the optimization.
    b. Test other values for the elbow angle where the results are likely to change.

```python
def cf_f1(x):
    """Cost function: sum of forces."""
    return x[0] + x[1] + x[2]

def cf_f2(x):
    """Cost function: sum of forces squared."""
    return x[0]**2 + x[1]**2 + x[2]**2

def cf_fpcsa2(x, a):
    """Cost function: sum of squared muscle stresses."""
    return (x[0]/a[0])**2 + (x[1]/a[1])**2 + (x[2]/a[2])**2

def cf_fmmax3(x, m):
    """Cost function: sum of cubic forces normalized by moments."""
    return (x[0]/m[0])**3 + (x[1]/m[1])**3 + (x[2]/m[2])**3
def cf_f1d(x):
    """Derivative of cost function: sum of forces."""
    dfdx0 = 1
    dfdx1 = 1
    dfdx2 = 1
    return np.array([dfdx0, dfdx1, dfdx2])

def cf_f2d(x):
    """Derivative of cost function: sum of forces squared."""
    dfdx0 = 2*x[0]
    dfdx1 = 2*x[1]
    dfdx2 = 2*x[2]
    return np.array([dfdx0, dfdx1, dfdx2])

def cf_fpcsa2d(x, a):
    """Derivative of cost function: sum of squared muscle stresses."""
    dfdx0 = 2*x[0]/a[0]**2
    dfdx1 = 2*x[1]/a[1]**2
    dfdx2 = 2*x[2]/a[2]**2
    return np.array([dfdx0, dfdx1, dfdx2])

def cf_fmmax3d(x, m):
    """Derivative of cost function: sum of cubic forces normalized by moments."""
    dfdx0 = 3*x[0]**2/m[0]**3
    dfdx1 = 3*x[1]**2/m[1]**3
    dfdx2 = 3*x[2]**2/m[2]**3
    return np.array([dfdx0, dfdx1, dfdx2])
```

```python
# time elbow_flexion BIClong BICshort BRA
r_ef = np.loadtxt('./../../../data/r_elbowflexors.mot', skiprows=7)
f_ef = np.loadtxt('./../../../data/f_elbowflexors.mot', skiprows=7)

m_ef = r_ef*1
m_ef[:, 2:] = r_ef[:, 2:]*f_ef[:, 2:]

a_ef = np.array([624.3, 435.56, 987.26])/50  # 50 N/cm2
```

```python
M = 20  # desired torque at the elbow
iang = 69  # which will give the closest value to 90 degrees

r  = r_ef[iang, 2:]
f0 = f_ef[iang, 2:]
a  = a_ef
m  = m_ef[iang, 2:]
x0 = f_ef[iang, 2:]/235  # far from the correct answer for the sum of torques
print('M =', M)
print('x0 =', x0)
print('r * x0 =', np.sum(r*x0))

bnds = ((0, f0[0]), (0, f0[1]), (0, f0[2]))

```

    M = 20
    x0 = [2.44736654 1.5446698  3.8147662 ]
    r * x0 = 0.28178742323694983

```python
# use this in combination with the parameter bounds:
cons = ({'type': 'eq',
         'fun' : lambda x, r, f0, M: np.array([r[0]*x[0] + r[1]*x[1] + r[2]*x[2] - M]),
         'jac' : lambda x, r, f0, M: np.array([r[0], r[1], r[2]]), 'args': (r, f0, M)})
# to enter everything as constraints:
cons = ({'type': 'eq',
         'fun' : lambda x, r, f0, M: np.array([r[0]*x[0] + r[1]*x[1] + r[2]*x[2] - M]),
         'jac' : lambda x, r, f0, M: np.array([r[0], r[1], r[2]]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: f0[0]-x[0],
         'jac' : lambda x, r, f0, M: np.array([-1, 0, 0]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: f0[1]-x[1],
         'jac' : lambda x, r, f0, M: np.array([0, -1, 0]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: f0[2]-x[2],
         'jac' : lambda x, r, f0, M: np.array([0, 0, -1]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: x[0],
         'jac' : lambda x, r, f0, M: np.array([1, 0, 0]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: x[1],
         'jac' : lambda x, r, f0, M: np.array([0, 1, 0]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: x[2],
         'jac' : lambda x, r, f0, M: np.array([0, 0, 1]), 'args': (r, f0, M)})
```

```python
f1r = minimize(fun=cf_f1, x0=x0, args=(), jac=cf_f1d,
               constraints=cons, method='SLSQP',
               options={'disp': True})
f2r = minimize(fun=cf_f2, x0=x0, args=(), jac=cf_f2d,
               constraints=cons, method='SLSQP',
               options={'disp': True})
fpcsa2r = minimize(fun=cf_fpcsa2, x0=x0, args=(a,), jac=cf_fpcsa2d,
                   constraints=cons, method='SLSQP',
                   options={'disp': True})
fmmax3r = minimize(fun=cf_fmmax3, x0=x0, args=(m,), jac=cf_fmmax3d,
                   constraints=cons, method='SLSQP',
                   options={'disp': True})
```

    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 409.59266009952
                Iterations: 7
                Function evaluations: 7
                Gradient evaluations: 7
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 75657.38479127164
                Iterations: 4
                Function evaluations: 6
                Gradient evaluations: 4
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 529.9639777695752
                Iterations: 11
                Function evaluations: 11
                Gradient evaluations: 11
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 1075.1388931095582
                Iterations: 12
                Function evaluations: 13
                Gradient evaluations: 12

```python
dat = np.vstack((np.around(r*100,1), np.around(a,1), np.around(f0,0), np.around(m,1)))
opt = np.around(np.vstack((f1r.x, f2r.x, fpcsa2r.x, fmmax3r.x)), 1)
er = ['-', '-', '-', '-',
      np.sum(r*f1r.x)-M, np.sum(r*f2r.x)-M, np.sum(r*fpcsa2r.x)-M, np.sum(r*fmmax3r.x)-M]
data = np.vstack((np.vstack((dat, opt)).T, er)).T

rows = ['$\text{Moment arm}\;[cm]$', '$pcsa\;[cm^2]$', '$F_{max}\;[N]$', '$M_{max}\;[Nm]$',
        '$\sum F_i$', '$\sum F_i^2$', '$\sum(F_i/pcsa_i)^2$', '$\sum(F_i/M_{max,i})^3$']
cols = ['Biceps long head', 'Biceps short head', 'Brachialis', 'Error in M']
df = pd.DataFrame(data, index=rows, columns=cols)
print('\nComparison of different cost functions for solving the distribution problem')
df
```

    Comparison of different cost functions for solving the distribution problem

      Biceps long head
      Biceps short head
      Brachialis
      Error in M

      $\text{Moment arm}\;[cm]$
      4.9
      4.9
      2.3
      -

      $pcsa\;[cm^2]$
      12.5
      8.7
      19.7
      -

      $F_{max}\;[N]$
      575.0
      363.0
      896.0
      -

      $M_{max}\;[Nm]$
      28.1
      17.7
      20.4
      -

      $\sum F_i$
      205.2
      204.3
      -0.0
      0.0

      $\sum F_i^2$
      184.7
      184.7
      86.1
      -3.552713678800501e-15

      $\sum(F_i/pcsa_i)^2$
      201.7
      98.2
      235.2
      0.0

      $\sum(F_i/M_{max,i})^3$
      241.1
      120.9
      102.0
      -3.552713678800501e-15

```python
M = 20  # desired torque at the elbow
iang = 35  # which will give the closest value to 90 degrees

r  = r_ef[iang, 2:]
f0 = f_ef[iang, 2:]
a  = a_ef
m  = m_ef[iang, 2:]
x0 = f_ef[iang, 2:]/235  # far from the correct answer for the sum of torques
print('M =', M)
print('x0 =', x0)
print('r * x0 =', np.sum(r*x0))

bnds = ((0, f0[0]), (0, f0[1]), (0, f0[2]))
```

    M = 20
    x0 = [2.65687365 1.84300054 4.17723792]
    r * x0 = 0.20841024709802292

```python
f1r = minimize(fun=cf_f1, x0=x0, args=(), jac=cf_f1d,
               constraints=cons, method='SLSQP',
               options={'disp': True})

f2r = minimize(fun=cf_f2, x0=x0, args=(), jac=cf_f2d,
               constraints=cons, method='SLSQP',
               options={'disp': True})

fpcsa2r = minimize(fun=cf_fpcsa2, x0=x0, args=(a,), jac=cf_fpcsa2d,
                   constraints=cons, method='SLSQP',
                   options={'disp': True})

fmmax3r = minimize(fun=cf_fmmax3, x0=x0, args=(m,), jac=cf_fmmax3d,
                   constraints=cons, method='SLSQP',
                   options={'disp': True})
```

    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 409.59266009953393
                Iterations: 7
                Function evaluations: 7
                Gradient evaluations: 7
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 75657.3847912717
                Iterations: 4
                Function evaluations: 6
                Gradient evaluations: 4
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 529.9639777695751
                Iterations: 11
                Function evaluations: 11
                Gradient evaluations: 11
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 2330.3142947977667
                Iterations: 10
                Function evaluations: 10
                Gradient evaluations: 10

```python
dat = np.vstack((np.around(r*100,1), np.around(a,1), np.around(f0,0), np.around(m,1)))
opt = np.around(np.vstack((f1r.x, f2r.x, fpcsa2r.x, fmmax3r.x)), 1)
er = ['-', '-', '-', '-',
      np.sum(r*f1r.x)-M, np.sum(r*f2r.x)-M, np.sum(r*fpcsa2r.x)-M, np.sum(r*fmmax3r.x)-M]
data = np.vstack((np.vstack((dat, opt)).T, er)).T

rows = ['$\text{Moment arm}\;[cm]$', '$pcsa\;[cm^2]$', '$F_{max}\;[N]$', '$M_{max}\;[Nm]$',
        '$\sum F_i$', '$\sum F_i^2$', '$\sum(F_i/pcsa_i)^2$', '$\sum(F_i/M_{max,i})^3$']
cols = ['Biceps long head', 'Biceps short head', 'Brachialis', 'Error in M']
df = pd.DataFrame(data, index=rows, columns=cols)
print('\nComparison of different cost functions for solving the distribution problem')
df
```

    Comparison of different cost functions for solving the distribution problem

      Biceps long head
      Biceps short head
      Brachialis
      Error in M

      $\text{Moment arm}\;[cm]$
      3.4
      3.4
      1.3
      -

      $pcsa\;[cm^2]$
      12.5
      8.7
      19.7
      -

      $F_{max}\;[N]$
      624.0
      433.0
      982.0
      -

      $M_{max}\;[Nm]$
      21.5
      14.9
      12.5
      -

      $\sum F_i$
      205.2
      204.4
      0.0
      -5.875094718302668

      $\sum F_i^2$
      184.7
      184.7
      86.1
      -6.162588012268191

      $\sum(F_i/pcsa_i)^2$
      201.7
      98.2
      235.2
      -6.660232419069558

      $\sum(F_i/M_{max,i})^3$
      238.3
      137.7
      72.1
      -6.115622167519737

```python

```

4. In an experiment to estimate forces of the elbow flexors, through inverse dynamics it was found an elbow flexor moment of 10 Nm.
Consider the following data for maximum force (F0), moment arm (r), and pcsa (A) of the brachialis, brachioradialis, and biceps brachii muscles: F0 (N): 1000, 250, 700; r (cm): 2, 5, 4; A (cm$^2$): 33, 8, 23, respectively (data from Robertson et al. (2013)).
    a. Use static optimization to estimate the muscle forces.
    b. Test the robustness of the results using different initial values for the muscle forces.
    c. Compare the results for different cost functions.

```python
M = 10  # desired torque at the elbow

f0 = np.array([1000, 250, 700])
r  = np.array([2, 5, 4])
a  = np.array([33, 8, 23])

m = r*f0
x0 = f0*10  # far from the correct answer for the sum of torques
print('M =', M)
print('x0 =', x0)
print('r * x0 =', np.sum(r*x0))

bnds = ((0, f0[0]), (0, f0[1]), (0, f0[2]))
```

    M = 10
    x0 = [10000  2500  7000]
    r * x0 = 60500

```python
# use this in combination with the parameter bounds:
cons = ({'type': 'eq',
         'fun' : lambda x, r, f0, M: np.array([r[0]*x[0] + r[1]*x[1] + r[2]*x[2] - M]),
         'jac' : lambda x, r, f0, M: np.array([r[0], r[1], r[2]]), 'args': (r, f0, M)})
# to enter everything as constraints:
cons = ({'type': 'eq',
         'fun' : lambda x, r, f0, M: np.array([r[0]*x[0] + r[1]*x[1] + r[2]*x[2] - M]),
         'jac' : lambda x, r, f0, M: np.array([r[0], r[1], r[2]]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: f0[0]-x[0],
         'jac' : lambda x, r, f0, M: np.array([-1, 0, 0]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: f0[1]-x[1],
         'jac' : lambda x, r, f0, M: np.array([0, -1, 0]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: f0[2]-x[2],
         'jac' : lambda x, r, f0, M: np.array([0, 0, -1]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: x[0],
         'jac' : lambda x, r, f0, M: np.array([1, 0, 0]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: x[1],
         'jac' : lambda x, r, f0, M: np.array([0, 1, 0]), 'args': (r, f0, M)},
        {'type': 'ineq', 'fun' : lambda x, r, f0, M: x[2],
         'jac' : lambda x, r, f0, M: np.array([0, 0, 1]), 'args': (r, f0, M)})
```

```python
f1r = minimize(fun=cf_f1, x0=x0, args=(), jac=cf_f1d,
               constraints=cons, method='SLSQP',
               options={'disp': True})
f2r = minimize(fun=cf_f2, x0=x0, args=(), jac=cf_f2d,
               constraints=cons, method='SLSQP',
               options={'disp': True})
fpcsa2r = minimize(fun=cf_fpcsa2, x0=x0, args=(a,), jac=cf_fpcsa2d,
                   constraints=cons, method='SLSQP',
                   options={'disp': True})
fmmax3r = minimize(fun=cf_fmmax3, x0=x0, args=(m,), jac=cf_fmmax3d,
                   constraints=cons, method='SLSQP',
                   options={'disp': True})
```

    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 1.9999999999999882
                Iterations: 5
                Function evaluations: 5
                Gradient evaluations: 5
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 2.2222222227876145
                Iterations: 6
                Function evaluations: 7
                Gradient evaluations: 6
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 0.006934901666808587
                Iterations: 14
                Function evaluations: 14
                Gradient evaluations: 14
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 1.5623481833790065e-08
                Iterations: 2
                Function evaluations: 3
                Gradient evaluations: 2

```python
dat = np.vstack((np.around(r*100,1), np.around(a,1), np.around(f0,0), np.around(m,1)))
opt = np.around(np.vstack((f1r.x, f2r.x, fpcsa2r.x, fmmax3r.x)), 1)
er = ['-', '-', '-', '-',
      np.sum(r*f1r.x)-M, np.sum(r*f2r.x)-M, np.sum(r*fpcsa2r.x)-M, np.sum(r*fmmax3r.x)-M]
data = np.vstack((np.vstack((dat, opt)).T, er)).T

rows = ['$\text{Moment arm}\;[cm]$', '$pcsa\;[cm^2]$', '$F_{max}\;[N]$', '$M_{max}\;[Nm]$',
        '$\sum F_i$', '$\sum F_i^2$', '$\sum(F_i/pcsa_i)^2$', '$\sum(F_i/M_{max,i})^3$']
cols = ['Biceps long head', 'Biceps short head', 'Brachialis', 'Error in M']
df = pd.DataFrame(data, index=rows, columns=cols)
print('\nComparison of different cost functions for solving the distribution problem')
df
```

    Comparison of different cost functions for solving the distribution problem

      Biceps long head
      Biceps short head
      Brachialis
      Error in M

      $\text{Moment arm}\;[cm]$
      200.0
      500.0
      400.0
      -

      $pcsa\;[cm^2]$
      33.0
      8.0
      23.0
      -

      $F_{max}\;[N]$
      1000.0
      250.0
      700.0
      -

      $M_{max}\;[Nm]$
      2000.0
      1250.0
      2800.0
      -

      $\sum F_i$
      -0.0
      2.0
      -0.0
      -1.7763568394002505e-15

      $\sum F_i^2$
      0.4
      1.1
      0.9
      0.0

      $\sum(F_i/pcsa_i)^2$
      1.5
      0.2
      1.5
      0.0

      $\sum(F_i/M_{max,i})^3$
      5.0
      0.0
      0.0
      0.0

```python
M = 10  # desired torque at the elbow

f0 = np.array([1000, 250, 700])
r  = np.array([2, 5, 4])
a  = np.array([33, 8, 23])

m = r*f0
x0 = f0/235  # far from the correct answer for the sum of torques
print('M =', M)
print('x0 =', x0)
print('r * x0 =', np.sum(r*x0))

bnds = ((0, f0[0]), (0, f0[1]), (0, f0[2]))
```

    M = 10
    x0 = [4.25531915 1.06382979 2.9787234 ]
    r * x0 = 25.744680851063826

```python
f1r = minimize(fun=cf_f1, x0=x0, args=(), jac=cf_f1d,
               constraints=cons, method='SLSQP',
               options={'disp': True})
f2r = minimize(fun=cf_f2, x0=x0, args=(), jac=cf_f2d,
               constraints=cons, method='SLSQP',
               options={'disp': True})
fpcsa2r = minimize(fun=cf_fpcsa2, x0=x0, args=(a,), jac=cf_fpcsa2d,
                   constraints=cons, method='SLSQP',
                   options={'disp': True})
fmmax3r = minimize(fun=cf_fmmax3, x0=x0, args=(m,), jac=cf_fmmax3d,
                   constraints=cons, method='SLSQP',
                   options={'disp': True})
```

    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 2.000000000000004
                Iterations: 6
                Function evaluations: 6
                Gradient evaluations: 6
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 2.222222231479426
                Iterations: 7
                Function evaluations: 7
                Gradient evaluations: 7
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 0.006934812760199668
                Iterations: 11
                Function evaluations: 11
                Gradient evaluations: 11
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 4.177724714444361e-09
                Iterations: 1
                Function evaluations: 2
                Gradient evaluations: 1

```python
dat = np.vstack((np.around(r*100,1), np.around(a,1), np.around(f0,0), np.around(m,1)))
opt = np.around(np.vstack((f1r.x, f2r.x, fpcsa2r.x, fmmax3r.x)), 1)
er = ['-', '-', '-', '-',
      np.sum(r*f1r.x)-M, np.sum(r*f2r.x)-M, np.sum(r*fpcsa2r.x)-M, np.sum(r*fmmax3r.x)-M]
data = np.vstack((np.vstack((dat, opt)).T, er)).T

rows = ['$\text{Moment arm}\;[cm]$', '$pcsa\;[cm^2]$', '$F_{max}\;[N]$', '$M_{max}\;[Nm]$',
        '$\sum F_i$', '$\sum F_i^2$', '$\sum(F_i/pcsa_i)^2$', '$\sum(F_i/M_{max,i})^3$']
cols = ['Biceps long head', 'Biceps short head', 'Brachialis', 'Error in M']
df = pd.DataFrame(data, index=rows, columns=cols)
print('\nComparison of different cost functions for solving the distribution problem')
df
```

    Comparison of different cost functions for solving the distribution problem

      Biceps long head
      Biceps short head
      Brachialis
      Error in M

      $\text{Moment arm}\;[cm]$
      200.0
      500.0
      400.0
      -

      $pcsa\;[cm^2]$
      33.0
      8.0
      23.0
      -

      $F_{max}\;[N]$
      1000.0
      250.0
      700.0
      -

      $M_{max}\;[Nm]$
      2000.0
      1250.0
      2800.0
      -

      $\sum F_i$
      0.0
      2.0
      0.0
      0.0

      $\sum F_i^2$
      0.4
      1.1
      0.9
      0.0

      $\sum(F_i/pcsa_i)^2$
      1.5
      0.2
      1.5
      3.552713678800501e-15

      $\sum(F_i/M_{max,i})^3$
      3.2
      0.0
      0.9
      1.0658141036401503e-14

```python

```


**1**. Making wallpaper with `fromfunction`

Adapted from [Circle Squared](http://igpphome.ucsd.edu/~shearer/COMP233/SciAm_Mandel.pdf)

Create a $400 \times 400$ array using the function `lambda i, j:  0.27**2*(i**2 + j**2) % 1.5`. Use `imshow` from `matplotlib.pyplot` with `interpolation='nearest'` and the `YlOrBr` colormap to display the resulting array as an image.

```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
```

```python
xy = np.fromfunction(lambda i, j: 0.27**2*(i**2 + j**2) % 1.5, (400, 400))
plt.figure(figsize = (8, 8))
plt.imshow(xy, interpolation = 'nearest', cmap = plt.cm.YlOrBr)
```

**2**. Find t least squares solution for $\beta_0, \beta_1, \beta_2$ using the normal equations $\hat{\beta} = (X^TX)^{-1}x^Ty$.

\begin{align}
10 &= \beta_0 + 3 \beta_1  + 7 \beta_2 \\
11 &= \beta_0 + 2 \beta_1  + 8 \beta_2 \\
9 &= \beta_0 + 3 \beta_1 + 7 \beta_2 \\
10 &= \beta_0 + 1 \beta_1 + 9 \beta_2 \\
\end{align}

You can find the inverse of a matrix by using `np.linalg.inv` and the transpose with `X.T`

```python
X = np.c_[np.ones(4), [3,2,3,1], [7,8,7,9]]
y = np.array([10,11,9,10]).reshape(4,1)
```

```python
b = np.linalg.inv(X.T @ X) @ X.T @ y
b
```

    array([[-1280.],
           [    0.],
           [   29.]])

```python
b2 = np.linalg.solve(X.T @ X,  X.T @ y)
b2
```

    array([[ 9.10878011],
           [-0.19269619],
           [ 0.17094017]])

```python
np.c_[y, X @ b]
```

    array([[   10., -1077.],
           [   11., -1048.],
           [    9., -1077.],
           [   10., -1019.]])

```python
np.c_[y, X @ b2]
```

    array([[10.        ,  9.72727273],
           [11.        , 10.09090909],
           [ 9.        ,  9.72727273],
           [10.        , 10.45454545]])


# 1. Perceptron
**Single Layer Perceptron(one Neuron)**
A Perceptron is a simple model of a biological neuron that classifies the label of the input data based on various activation functions:
1. Binary Step Function:
\begin{equation}
  f(x)=\begin{cases}
    0, & \text{if $x<0$}.\\
    1, & \text{otherwise}.
  \end{cases}
\end{equation}

2. Signum Function:
\begin{equation}
  f(x)=\begin{cases}
    -1, & \text{if $x<0$}.\\
    0, & \text{if $x=0$>}\\
    1, & \text{if $x>0$}.
  \end{cases}
\end{equation}

3. Linear Activation Function:
$$f(x) = x$$

4. Sigmoid / Logistic Activation Function:
$$f(x) = \frac{1}{1+e^{-x}}$$

5. Tanh Function (Hyperbolic Tangent):
$$f(x) = \frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}$$

6. ReLU Function:
$$f(x) = max(0, x)$$

7. Exponential Linear Units (ELUs) Function:
\begin{equation}
  f(x)=\begin{cases}
    x, & \text{if $x\geqslant0$}.\\
    \alpha(e^{x} - 1), & \text{otherwise}.
  \end{cases}
\end{equation}

8. Swish:
$$f(x) = \frac{x}{1 + e^{-x}}$$

9. Gaussian Error Linear Unit (GELU):
$$f(x) = 0.5x(1 + tanh[\sqrt{2/\pi}(x + 0.044715x^3)])$$

```python
fn_list = ['step', 'signum', 'linear', 'relu', 'sigmoid', 'tanh', 'elu', 'gelu', 'swish']
```

## 1.1. How to use this
```python
from Perceptron.perceptron import Perceptron
from Perceptron.utils import prepare_data, save_plot, save_model

# get the data, convert it into a DataFrame and then use below commands
X, y = prepare_data(df)

model = Perceptron(eta = eta, epochs = epochs)
model.fit(X, y, fn, alpha=None) # alpha ranges between 0 to 1 if and only if ELU activation function is applied else alpha value remains None for other activation functions

Total_Error = model.total_loss()

save_model(model, filename = filename)

save_plot(df, plotFilename, model)
```

## 1.2. Reference
[Python Package Publishing Docs](https://packaging.python.org/tutorials/packaging-projects/)

[GitHub Actions CICD Docs](https://docs.github.com/en/actions/guides/building-and-testing-python#publishing-to-package-registries)


```python
from sympy import *
from sympy.solvers.solveset import solveset
init_printing()
x, y, z = symbols('x,y,z')
```

## Solveset

Equation solving is both a common need also a common building block for more complicated symbolic algorithms.

Here we introduce the `solveset` function.

```python
solveset(x**2 - 4, x)
```

Solveset takes two arguments and one optional argument specifying the domain, an equation like $x^2 - 4$ and a variable on which we want to solve, like $x$ and an optional argument domain specifying the region in which we want to solve.

Solveset returns the values of the variable, $x$, for which the equation, $x^2 - 4$ equals 0.

### Exercise

What would the following code produce?  Are you sure?

```python
solveset(x**2 - 9 == 0, x)
```

## Infinite Solutions

One of the major improvements of `solveset` is that it also supports infinite solution.

```python
solveset(sin(x), x)
```

## Domain argument

```python
solveset(exp(x) -1, x)
```

`solveset` by default solves everything in the complex domain. In complex domain $exp(x) == cos(x) + i\ sin(x)$ and solution is basically equal to solution to $cos(x) == 1$. If you want only real solution, you can specify the domain as `S.Reals`.

```python
solveset(exp(x) -1, x, domain=S.Reals)
```

## Condition Set

`solveset` isn't always able to solve a given equation, such cases it returns a `ConditionSet` object. `ConditionSet` represents a set satisfying a given condition.

```python
solveset(exp(x) + cos(x) + 1, x, domain=S.Reals)
```

`solveset` aims to return all the solutions of the equation. In cases where it able to find some solution but not all it returns a union of the known solutions and `ConditionSet`.

```python
solveset((x - 1)*(exp(x) + cos(x) + 1), x, domain=S.Reals)
```

## Symbolic use of `solveset`

Results of `solveset` don't need to be numeric, like `{-2, 2}`.  We can use solveset to perform algebraic manipulations.  For example if we know a simple equation for the area of a square

    area = height * width

we can solve this equation for any of the variables.  For example how would we solve this system for the `height`, given the `area` and `width`?

```python
height, width, area = symbols('height, width, area')
solveset(area - height*width, height)
```

Note that we would have liked to have written

    solveset(area == height * width, height)

But the `==` gotcha bites us.  Instead we remember that `solveset` expects an expression that is equal to zero, so we rewrite the equation

    area = height * width

into the equation

    0 = height * width - area

and that is what we give to solveset.

### Exercise

Compute the radius of a sphere, given the volume.  Reminder, the volume of a sphere of radius `r` is given by

$$ V = \frac{4}{3}\pi r^3 $$

```python
# Solve for the radius of a sphere, given the volume

```

You will probably get several solutions, this is fine.  The first one is probably the one that you want.

## Substitution

We often want to substitute in one expression for another.  For this we use the subs method

```python
x**2
```

```python
# Replace x with y
(x**2).subs({x: y})
```

### Exercise

Subsitute $x$ for $sin(x)$ in the equation $x^2 + 2\cdot x + 1$

```python
# Replace x with sin(x)

```

## Subs + Solveset

We can use subs and solve together to plug the solution of one equation into another

```python
# Solve for the height of a rectangle given area and width

soln = list(solveset(area - height*width, height))[0]
soln
```

```python
# Define perimeter of rectangle in terms of height and width

perimeter = 2*(height + width)
```

```python
# Substitute the solution for height into the expression for perimeter

perimeter.subs({height: soln})
```

### Exercise

In the last section you solved for the radius of a sphere given its volume

```python
V, r = symbols('V,r', real=True)
4*pi/3 * r**3
```

```python
list(solveset(V - 4*pi/3 * r**3, r))[0]
```

Now lets compute the surface area of a sphere in terms of the volume.  Recall that the surface area of a sphere is given by

$$ 4 \pi r^2 $$

```python
(?).subs(?)
```

Does the expression look right?  How would you expect the surface area to scale with respect to the volume?  What is the exponent on $V$?

## Plotting

SymPy can plot expressions easily using the `plot` function.  By default this links against matplotlib.

```python
%matplotlib inline
```

```python
plot(x**2)
```

### Exercise

In the last exercise you derived a relationship between the volume of a sphere and the surface area.  Plot this relationship using `plot`.

```python
plot(?)
```

## Low dependencies

You may know that SymPy tries to be a very low-dependency project.  Our user base is very broad.  Some entertaining aspects result.  For example, `textplot`.

```python
textplot(x**2, -3, 3)
```

### Exercise

Play with `textplot` and enjoy :)


# the $k$-nearest neighbors (kNN) supervised learning algorithm

* can be used for regression or classification
* non-parametric
* imposes only mild structural assumptions about the data

let's explore $k$-nearest neighbor classification.

training data examples: $\{(\mathbf{x}_1, y_1), (\mathbf{x}_2, y_2), ..., (\mathbf{x}_n, y_n)\}$ where $\mathbf{x}_i$ is the feature vector and $y_i \in \{0, 1\}$ is the label on data point $i$. This is a binary classification since each data point is labeled with $0$ or $1$.

the $k$-NN algorithm uses the training data to classify new data points. say we have a new data point $\mathbf{x}$ but we don't know its label. the $k$-NN algorithm predicts the class of this data point as:
\begin{equation}
    \hat{y}(\mathbf{x})= \frac{1}{k} \displaystyle \sum_{\mathbf{x}_i \in N_k(\mathbf{x})} y_i
\end{equation}
where $N_k(x)$ is the *neighborhood* of $\mathbf{x}$ defined as the $k$ "closest" points in the training data set. For "closest" to be mathematically defined, we need a distance metric. One such distance metric is Euclidean distance. In words, the $k$ data points closest to $\mathbf{x}$ vote on whether we should classify this point as a 0 or 1 with their labels.

```julia
using CSV
using DataFrames
using PyPlot
using ScikitLearn # machine learning package
using StatsBase
using Random
using LaTeXStrings # for L"$x$" to work instead of needing to do "\$x\$"
using Printf

# (optional)change settings for all plots at once, e.g. font size
rcParams = PyPlot.PyDict(PyPlot.matplotlib."rcParams")
rcParams["font.size"] = 16

# (optional) change the style. see styles here: https://matplotlib.org/3.1.1/gallery/style_sheets/style_sheets_reference.html
PyPlot.matplotlib.style.use("seaborn-white")
```

## classifying breast tumors as malignant or benign

source: [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic))

> Features are computed from a digitized image of a fine needle aspirate (FNA) of a breast mass. They describe characteristics of the cell nuclei present in the image.

The mean radius and smoothness of the cell nuclei (the two features) and the outcome (M = malignant, B = benign) of the tumor are in the `breast_cancer_data.csv`.

```julia
df = CSV.read("breast_cancer_data.csv")
first(df, 5)
```

	mean_radius	mean_smoothness	outcome

	Float64	Float64	String

5 rows × 3 columns

1	13.85	1.495	B

2	9.668	2.275	B

3	9.295	2.388	B

4	19.69	4.585	M

5	9.755	1.243	B

```julia

```

let's map the outcomes to a number, 0 or 1, to facilitate the NN algo.

benign (B) : 0

malignant (M) : 1

```julia

```

let's also have a color scheme for the class labels.

```julia

```

since this is a two-dimensional feature space, we have the luxury of visualizing how the classes are distributed in feature space. in practice, we do not have this luxury. I choose a 2D feature space for pedagogical purposes :)

```julia

```

### using Scikitlearn for $k$-nearest neighbor classification

scikitlearn documentation for `KNeighborsClassifier` [here](https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html)

```julia
@sk_import neighbors : KNeighborsClassifier
@sk_import model_selection : train_test_split
@sk_import metrics : confusion_matrix
```

    ┌ Warning: Module model_selection has been ported to Julia - try `import ScikitLearn: CrossValidation` instead
    └ @ ScikitLearn.Skcore /home/cokes/.julia/packages/ScikitLearn/bo2Pt/src/Skcore.jl:140

    PyObject <function confusion_matrix at 0x7fe40f6b6c80>

scikitlearn takes as input:
* a feature matrix `X`, which must be `n_samples` by `n_features`
* a target vector `y`, which must be `n_samples` long (of course)

```julia

```

construct a nearest neighbor object in scikitlearn

```julia

```

pass the `KNeighborsClassifier` object our training data. this will be used to make predictions on future, unseen data.

```julia

```

we can now make predictions on new, unseen data. that is, if we know the mean radius and mean smoothness of a new tumor, we can make a prediction about whether this tumor is malignant or benign. let's imagine we take measurements on a new tumor and it has the feature vector `x_new` below. we aim to predict whether it is benign or malignant.

```julia
# new tumor. unknown if malignant or benign...
x_new = [10.0 1.5] # should be B
x_new = [20.0 5.0] # should be M
```

    1×2 Array{Float64,2}:
     20.0  5.0

first, let's plot where it falls in feature space

```julia

```

predict whether this new tumor is benign or malignant

```julia

```

visualize the decision boundary

```julia

```

```julia
figure()
xlabel("mean radius")
ylabel("mean smoothness")
```

we can compute the accuracy of the prediction on the training data. but when $k=1$ we get 100% accuracy by construction!

```julia

```

### the test/train paradigm

randomly split your data set into a training and test set. train the $k$-NN algo on the training data, then use the trained model to make predictions on data in the test set. we can compare the predicted label to the true label. the test set error is a quality prediction of generalization error on *unseen* data. we could write our own code for this, but scikitlearn provides `test_train_split` for us.

```julia

```

to fully assess the performance of our model, trained on the training set, and tested on the test set, we can plot a so-called confusion matrix. it tells us about false positives, true positives, false negatives, and true negatives.

```julia
# see https://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py
function plot_confusion_matrix(y_true::Array{Float64}, y_pred::Array{Float64}, classes::Array{String})
    # calculate confusion matrix from scikitlearn
    cm = confusion_matrix(y_true, y_pred)

    fig, ax = plt.subplots()
    ax.imshow(cm, interpolation="nearest", cmap=plt.cm.Greens)
    ax.set(
        xlim=[-0.5, 1.5],
        ylim=[-0.5, 1.5],
        xticks=[0, 1],
        yticks=[0, 1],
        xticklabels=classes,
        yticklabels=classes,
        xlabel="predicted label",
        ylabel="true label"
    )

    for i = 1:2
        for j = 1:2
            ax.text(j-1, i-1, @sprintf("%d", cm[i, j]), ha="center", va="center")
        end
    end
    tight_layout()
end
```

    plot_confusion_matrix (generic function with 1 method)

to find the best $k$, we can scan over all $k$, train the model in the training set, then test the model on the test set (unseen data) to see how it performs. we take the "best" $k$ as the one that yields the lowest test set error, as we expect it to have the lowest generalization error on unseen data.

```julia

```

the optimal $k$ is around 8. So we should train our $k$-NN algo with $k=8$, then deploy this model, since it should result in the best accuracy on unseen data.

```julia

```


# 3.2.2 The Gauss–Markov Theorem

The Gauss-Markov theorem states that the least squares estimates of the $\beta$ have the smallest variance among all linear unbiased estimates. We focus on estimation of any linear combination of the parameters $\theta=\alpha^T\hat{\beta}$, i.e predictions $f(x_0)={x_0}^T\beta$ are of this form.
The least squares estimate is (3.17):

$$
\hat{\theta}=\alpha^T\hat{\beta} = \alpha^T(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{y}
$$

and $\alpha^T\hat{\beta}$ is unbiased, since (3.18):

$$
\begin{align}
E(\alpha^T\hat{\beta}) &= E( \alpha^T(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{y})\\
&=  \alpha^T(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T\mathbf{X}\beta\\
&= \alpha^T\beta
\end{align}
$$

The *Gauss-Markov* states that if we have any other linear estimator $\tilde{\theta}=\mathbf{c}^T\mathbf{y}$, that is, $E(\mathbf{c}^T\mathbf{y})=\alpha^T\beta$, then (3.19):

$$
Var(\alpha^T\hat{\beta})\le Var(\mathbf{c}^T\mathbf{y})
$$

*Proof*:

Let's assume that $\mathbf{c}^T\mathbf{y}=(\alpha^T(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T + \mathbf{d}^T)\mathbf{y}$, then:

$$
\begin{align}
E(\mathbf{c}^T\mathbf{y})&= E((\alpha^T(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T + \mathbf{d}^T)\mathbf{y})\\
&= E((\alpha^T(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T + \mathbf{d}^T)(\mathbf{X}\beta + \varepsilon))\\
&= \alpha^T\beta + d^T\mathbf{X}\beta
\end{align}
$$

Since $E(\mathbf{c}^T\mathbf{y})$ is unbiased $d^T\mathbf{X}=0$.

$$
\begin{align}
Var(\mathbf{c}^T\mathbf{y}) &= \mathbf{c}^{T}Var(\mathbf{y})\mathbf{c} = \sigma^2\mathbf{c}^T\mathbf{c}\\
&= \sigma^2(\alpha^T(\mathbf{X}^T\mathbf{X})^{-1}\mathbf{X}^T + \mathbf{d}^T)(\mathbf{X}(\mathbf{X}^T\mathbf{X})^{-1}\alpha + \mathbf{d})\\
&= \sigma^2(\alpha^T(\mathbf{X}^T\mathbf{X})^{-1}\alpha+\mathbf{d}^T\mathbf{d})\\
&= Var(\alpha^T\hat{\beta})+\sigma^2\mathbf{d}^T\mathbf{d}
\end{align}
$$

Since $\mathbf{d}^T\mathbf{d}\ge 0$, the proof is completed.

Consider the MSE of an estimator $\tilde{\theta}$ is estimating $\theta$:
$$
\begin{align}
MSE(\tilde{\theta}) &= E(\tilde{\theta} - \theta)^2\\
&= Var(\tilde{\theta})+[E(\tilde{\theta}) - \theta]^2
\end{align}
$$

The first term  is the variance, while the second term is the squared bias. The theorem implies that the least squares estimator has the smallest mean squared error of all linear estimators with no bias. However, there may exist a biased estimator with smaller MSE.

Mean squared error is related to prediction accuracy. Consider the prediction of the $x_0$, (3.21):

$$
Y_0 = f(x_0) + \varepsilon_0
$$

Then the EPE of an estimate $\tilde{f}(x_0)={x_0}^T\tilde{\beta}$ is (3.22):
$$
\begin{align}
E(Y_0 - \tilde{f}(x_0)) &= \sigma^2 + E({x_0}^T\tilde{\beta}-f(x_0))^2\\
&= \sigma^2 + MSE(\tilde{f}(x_0))
\end{align}
$$

Therefore, EPE and MSE differ only by the constant $\sigma^2$.


# Solutions

## Question 1

> `1`. Obtain the determinant and the inverses of the following matrices:

> `1`. $A = \begin{pmatrix} 1 / 5 & 1\\1 & 1\end{pmatrix}$

```python
import sympy as sym

A = sym.Matrix([[sym.S(1) / 5, 1], [1, 1]])
A.det()
```

$\displaystyle - \frac{4}{5}$

```python
A.inv()
```

$\displaystyle \left[\begin{matrix}- \frac{5}{4} & \frac{5}{4}\\\frac{5}{4} & - \frac{1}{4}\end{matrix}\right]$

> `2`. $B = \begin{pmatrix} 1 / 5 & 1 & 5\\3 & 1 & 6 \\ 1 & 2 & 1\end{pmatrix}$

```python
B = sym.Matrix([[sym.S(1) / 5, 1, 5], [3, 1, 6], [1, 2, 1]])
B.det()
```

$\displaystyle \frac{129}{5}$

```python
B.inv()
```

$\displaystyle \left[\begin{matrix}- \frac{55}{129} & \frac{15}{43} & \frac{5}{129}\\\frac{5}{43} & - \frac{8}{43} & \frac{23}{43}\\\frac{25}{129} & \frac{1}{43} & - \frac{14}{129}\end{matrix}\right]$

> `3`. $C = \begin{pmatrix} 1 / 5 & 5 & 5\\3 & 1 & 7 \\ a & b & c\end{pmatrix}$

```python
a, b, c = sym.Symbol("a"), sym.Symbol("b"), sym.Symbol("c")
C = sym.Matrix([[sym.S(1) / 5, 5, 5], [3, 1, 7], [a, b, c]])
C.det()
```

$\displaystyle 30 a + \frac{68 b}{5} - \frac{74 c}{5}$

```python
C.inv()
```

$\displaystyle \left[\begin{matrix}\frac{- 35 b + 5 c}{150 a + 68 b - 74 c} & \frac{25 b - 25 c}{150 a + 68 b - 74 c} & - \frac{444}{25 \left(- \frac{444 a}{25} - \frac{5032 b}{625} + \frac{5476 c}{625}\right)}\\\frac{35 a - 15 c}{150 a + 68 b - 74 c} & \frac{- 25 a + c}{150 a + 68 b - 74 c} & - \frac{5032}{125 \left(- \frac{444 a}{5} - \frac{5032 b}{125} + \frac{5476 c}{125}\right)}\\\frac{- 5 a + 15 b}{150 a + 68 b - 74 c} & \frac{25 a - b}{150 a + 68 b - 74 c} & - \frac{74}{25 \left(6 a + \frac{68 b}{25} - \frac{74 c}{25}\right)}\end{matrix}\right]$

## Question 2

> `2`. Compute the following:

> `1`. $500\begin{pmatrix} 1 / 5 & 1\\1 & 1\end{pmatrix}$

```python
A = 500 * sym.Matrix([[sym.S(1) / 5, 1], [1, 1]])
A
```

$\displaystyle \left[\begin{matrix}100 & 500\\500 & 500\end{matrix}\right]$

> `2`. $\pi \begin{pmatrix} 1 / \pi & 2\pi\\3/\pi & 1\end{pmatrix}$

```python
B = sym.pi * sym.Matrix([[1 / sym.pi, 2 * sym.pi], [3 / sym.pi, 1]])
B
```

$\displaystyle \left[\begin{matrix}1 & 2 \pi^{2}\\3 & \pi\end{matrix}\right]$

> `3`. $500\begin{pmatrix} 1 / 5 & 1\\1 & 1\end{pmatrix} + \pi \begin{pmatrix} 1 / \pi & 2\pi\\3/\pi & 1\end{pmatrix}$

```python
A + B
```

$\displaystyle \left[\begin{matrix}101 & 2 \pi^{2} + 500\\503 & \pi + 500\end{matrix}\right]$

> `4`. $500\begin{pmatrix} 1 / 5 & 1\\1 & 1\end{pmatrix}\begin{pmatrix} 1 / \pi & 2\pi\\3/\pi & 1\end{pmatrix}$

```python
C = sym.Matrix([[1 / sym.pi, 2 * sym.pi], [3 / sym.pi, 1]])
A @ C
```

$\displaystyle \left[\begin{matrix}\frac{1600}{\pi} & 500 + 200 \pi\\\frac{2000}{\pi} & 500 + 1000 \pi\end{matrix}\right]$

## Question 3

> `3`. The matrix $A$ is given by $A=\begin{pmatrix}a & 4 & 2\\ 1 & a & 0\\ 1 & 2 & 1\end{pmatrix}$.

> `1`. Find the determinant of $A$

```python
A = sym.Matrix([[a, 4, 2], [1, a, 0], [1, 2, 1]])
determinant = A.det()
determinant
```

$\displaystyle a^{2} - 2 a$

> `2`. Hence find the values of $a$ for which $A$ is singular.

$A$ is singular when the determinant is $0$ so we solve that equation:

```python
sym.solveset(determinant, a)
```

$\displaystyle \left\{0, 2\right\}$

> `3`. State, giving a brief reason in each case, whether the simultaneous equations
>
> $$
  \begin{array}{l}
          a x + 4y + 2z= 3a\\
          x + a y  = 1\\
          x + 2y + z = 3\\
  \end{array}
  $$

> have any solutions when:
> `1`. $a = 3$;

When $a$ is 3 the determinant is none zero, and so the matrix that represents
that linear system can be inverted.

> `2`. $a = 2$

When $a$ is 2 the determinant is zero and so the matrix that represents
that linear system cannot be inverted.

## Question 4

> `4`. The matrix $D$ is given by $D = \begin{pmatrix} a & 2 & 0\\ 3 & 1 & 2\\ 0 & -1 & 1\end{pmatrix}$ where $a\ne 2$.
> `1`. Find $D^{-1}$.

```python
D = sym.Matrix([[a, 2, 0], [3, 1, 2], [0, -1, 1]])
D_inverse = D.inv()
D_inverse
```

$\displaystyle \left[\begin{matrix}- \frac{9}{18 - 9 a} & \frac{6}{18 - 9 a} & - \frac{12}{18 - 9 a}\\\frac{3}{6 - 3 a} & - \frac{a}{6 - 3 a} & \frac{2 a}{6 - 3 a}\\- \frac{3}{3 a - 6} & \frac{a}{3 a - 6} & \frac{a - 6}{3 a - 6}\end{matrix}\right]$

> `2`. Hence of otherwise, solve the equations:
>
> $$
    \begin{array}{l}
        a x + 2y = 3\\
        3x + y + 2z = 4\\
        - y + z = 1\\
    \end{array}
    $$

This corresponds to calculating: $D^{-1} \begin{pmatrix}3\\4\\1\end{pmatrix}$

```python
b = sym.Matrix([[3], [4], [1]])
D_inverse @ b
```

$\displaystyle \left[\begin{matrix}- \frac{15}{18 - 9 a}\\- \frac{2 a}{6 - 3 a} + \frac{9}{6 - 3 a}\\\frac{4 a}{3 a - 6} + \frac{a - 6}{3 a - 6} - \frac{9}{3 a - 6}\end{matrix}\right]$


# Simple PageRank example

PageRank is a algorithm used by Google to rank the 'importance' of a web page. The concept is that every web page has an importance score, and it 'endows' its importance evenly amongst the pages to which it links. The will lead to a problem where the ranking of all pages is an eigenvalue problem. We will demonstrate the process for the small web of four pages.

Consider a web of four pages, $p_{0}$, $p_{1}$, $p_{2}$ and $p_{3}$. Consider the scenario:

- $p_{0}$ links to: $p_{1}$, $p_{2}$ and $p_{3}$
- $p_{1}$ links to: $p_{2}$ and $p_{3}$
- $p_{2}$ links to: $p_{0}$
- $p_{3}$ links to: $p_{0}$ and $p_{2}$

We can build a directed graph to describe the connections, and for this we use the package `networkx`. To visualise the graph, the packages `pygraphviz` and `pydot` are required.

```
import networkx as nx

# Create a directed networkx graph
G = nx.DiGraph()

# Add web pages (graph nodes)
for i in range(4):
   G.add_node(i, label="p" + str(i))

# Add outgoing web links with weights (directed graph edges)
G.add_edge(0, 1, weight=1.0/3.0, label="1/3")
G.add_edge(0, 2, weight=1.0/3.0, label="1/3")
G.add_edge(0, 3, weight=1.0/3.0, label="1/3")

G.add_edge(1, 2, weight=1.0/2.0, label="1/2")
G.add_edge(1, 3, weight=1.0/2.0, label="1/2")

G.add_edge(2, 0, weight=1.0, label="1")

G.add_edge(3, 0, weight=1.0/2.0, label="1/2")
G.add_edge(3, 2, weight=1.0/2.0, label="1/2")

# To plot graph, convert to a PyGraphviz graph for drawing
Ag = nx.to_agraph(G)
Ag.layout(prog='dot')
Ag.draw('web.png')
from IPython.display import Image
Image('web.png')
```

Note that edges have been given a weight which is $1/n$, where $n$ is the number of outgoing link/edges from a page/node. The is the fraction of page's importance/rank that it can give to the pages to which it links.

If we denote the rank/importance of page $i$ by $x_{i}$, we can express the importance of each page:

$$
\begin{align}
x_{0} =& x_{2} + \tfrac{1}{2}x_{2}
\\
x_{1} =& \tfrac{1}{3}x_{0}
\\
x_{2} =& \tfrac{1}{3}x_{0} + \tfrac{1}{2}x_{1} + \tfrac{1}{2}x_{3}
\\
x_{3} =& \tfrac{1}{3}x_{0} + \tfrac{1}{2}x_{1}
\end{align}
$$

We can express this as a system of equation:

$$
\underbrace{
\begin{bmatrix}
0 & 0 & 1 & \tfrac{1}{2}
\\
\tfrac{1}{3} & 0 & 0 & 0
\\
\tfrac{1}{3} & \tfrac{1}{2} & 0 & \tfrac{1}{2}
\\
\tfrac{1}{3} & \tfrac{1}{2} & 0 & 0
\end{bmatrix}}_{\boldsymbol{A}}
\begin{bmatrix}
x_{0} \\ x_{1} \\ x_{2} \\ x_{3}
\end{bmatrix}
=
\begin{bmatrix}
x_{0} \\ x_{1} \\ x_{2} \\ x_{3}
\end{bmatrix}
$$

This is an eigenvalue problem with eigenvalue $\lambda = 1$. To solve the problem we need to find the corresponding eigenvector.

First, we can create the matrix $\boldsymbol{A}$ directly from the graph (note that we added weights to the graph edges when building the graph):

```
# Get the matrix containing the graph weights
A = (nx.adjacency_matrix(G).T).getA()
print A
```

    [[ 0.          0.          1.          0.5       ]
     [ 0.33333333  0.          0.          0.        ]
     [ 0.33333333  0.5         0.          0.5       ]
     [ 0.33333333  0.5         0.          0.        ]]

Note that the columns of $\boldsymbol{A}$ sum to one. Such a matrix is known as a *stochastic matrix*. In our context, what is important is that the largest eigenvalue (in absolute terms) for such a matrix is one. Therefore, we are looking for the eigenvector associated with the largest eigenvalue.

## Direct computation of the PageRank vector

To find the solution, we can compute the eigenvectors of $\boldsymbol{A}$, and pick the eigenvectors that corresponds to $\lambda = 1$:

```
# Compute the eigenvalues and eigenvectors
import numpy as np
evalues, evectors = np.linalg.eig(A)

# Print largest eigenvalue and corresponding eigenvector
print("The maxiumum eigenvalue is: {}".format(np.max(evalues)))
evector = evectors[:, np.argmax(evalues)]
evector /= np.linalg.norm(evector, 1)
print("The PageRank vector (eigenvector) is: \n {}".format(evector))
```

    The maxiumum eigenvalue is: (1+0j)
    The PageRank vector (eigenvector) is:
     [ 0.38709677+0.j  0.12903226+0.j  0.29032258+0.j  0.19354839+0.j]

The PageRank vector has been normalised using the $l_{1}$ norm such the the entries sum to one.

## Approximation of the PageRank vector via matrix multiplication

The direct computation of all eigenvalues and eigenvectors is an expensive operation, and can only be reasonably performed for small matrices. However, we are interested only in the eigenvector corresponding to the largest eigenvalue. Recall that, in almost all cases, the repeated multiplication of a vector by a matrix $\boldsymbol{A}$ will yield a vector that tends to the direction of the eigenvector corresponding to the largest eigenvalue. We can use this to approximate the PageRank vector using only matrix-vector multiplication:

```
# Create random starting vector
x0 = np.random.rand(A.shape[0])

# Perform 5 iterations
for i in range(5):
    x0 = A.dot(x0)
x0 = x0/np.linalg.norm(x0 ,1)
print("Estimated PageRank after 5 iterations: {}".format(x0))

# Perform another 5 iterations
for i in range(5):
    x0 = A.dot(x0)
x0 = x0/np.linalg.norm(x0 ,1)
print("Estimated PageRank after 10 iterations: {}".format(x0))

```

    Estimated PageRank after 5 iterations: [ 0.41444044  0.12356961  0.2825718   0.17941816]
    Estimated PageRank after 10 iterations: [ 0.38821763  0.12815233  0.29052499  0.19310505]

It is clear that the approximate solution is very close to the exact solution with relatively few iterations. The major advantage of this method is that it is computationally inexpensive, which makes it tractable for very large matrices.


```python
from scipy.stats import norm
from scipy.stats import t
import numpy as np
import pandas as pd
from numpy.random import seed
import matplotlib.pyplot as plt
```

```python
#generates random variates to draw five samples from the standard normal distribution
sample1 = norm.rvs(size=5)
```

```python
np.mean(sample1)
```

    0.20789409052637584

```python
seed(47)
sample2 = norm.rvs(size=5)
sample2
```

    array([-0.84800948,  1.30590636,  0.92420797,  0.6404118 , -1.05473698])

```python
np.mean(sample2)
```

    0.19355593334131074

```python
#standard deviation - calculating manually
diff_squared = (sample2 - np.mean(sample2))**2
v = np.sum(diff_squared) / len(sample2)
print('Variance: ' + str(v))
print('Standard Deviation: ' + str(np.sqrt(v)))
```

    Variance: 0.9227899466393845
    Standard Deviation: 0.9606195639478641

```python
#standard deviation - calculating using np.sqrt(np.var())
print('Variance: ' + str(np.sqrt(np.var(sample2))))
```

    Variance: 0.9606195639478641

```python
#standard deviation - calculating using np.std()
print('Standard Deviation: ' + str(np.std(sample2)))
```

    Standard Deviation: 0.9606195639478641

```python
#standard deviation for sample
var_b = np.sum(diff_squared) / (len(sample2)-1)
np.sqrt(var_b)
```

    1.0740053227518152

```python
#with np.std() we get population standard deviation
np.std(sample2)
```

    0.9606195639478641

```python
#adding ddof=1 to np.std() will return the sample standard deviation
np.std(sample2, ddof=1)
```

    1.0740053227518152

For the sampling distribution of the mean, the standard deviation of this distribution is given by

\begin{equation}
\sigma_{mean} = \frac{\sigma}{\sqrt n}
\end{equation}

where $\sigma_{mean}$ is the standard deviation of the sampling distribution of the mean and $\sigma$ is the standard deviation of the population (the population parameter).

Let us imagine we live in a town of 50000 people and we know the height of everyone in this town. We will have 50000 numbers that tell us everything about our population. We'll simulate these numbers now and put ourselves in one particular town, called 'town 47', where the **population mean** height is **172** cm and **population standard deviation** is **5** cm.

```python
#Generate 50000 samples from the population heights using 172 population mean and 5 population std
seed(47)
pop_heights = norm.rvs(172, 5, size=50000)
pop_heights.min(), pop_heights.max(), np.mean(pop_heights), np.std(pop_heights)
```

    (150.86806415764312, 193.22214997313296, 172.0192602425845, 4.990839139566387)

```python
#Using histogram to show the distribution of heights in the population
_ = plt.hist(pop_heights, bins=30)
_ = plt.xlabel('height (cm)')
_ = plt.ylabel('number of people')
_ = plt.title('Distribution of heights in entire town population')
_ = plt.axvline(172, color='r')
_ = plt.axvline(172+5, color='r', linestyle='--')
_ = plt.axvline(172-5, color='r', linestyle='--')
_ = plt.axvline(172+10, color='r', linestyle='-.')
_ = plt.axvline(172-10, color='r', linestyle='-.')
```

Now, 50000 people is rather a lot to chase after with a tape measure. If all you want to know is the average height of the townsfolk, then can you just go out and measure a sample to get a pretty good estimate of the average height?

```python
#creating a function to create a sample size(n) from our population heights
def townsfolk_sampler(n):
    return np.random.choice(pop_heights, n)
```

Let's say you go out one day and randomly sample 10 people to measure.

```python
seed(47)
daily_sample1 = townsfolk_sampler(10)
```

```python
_ = plt.hist(daily_sample1, bins=10)
_ = plt.xlabel('height (cm)')
_ = plt.ylabel('number of people')
_ = plt.title('Distribution of heights in sample size 10')
```

The sample distribution doesn't look much like what we know (but wouldn't know in real-life) the population distribution looks like. What do we get for the mean?

```python
np.mean(daily_sample1)
```

    173.47911444163503

And if we went out and repeated this experiment?

```python
daily_sample2 = townsfolk_sampler(10)
```

```python
np.mean(daily_sample2)
```

    173.7317666636263

Simulate performing this random trial every day for a year, calculating the mean of each daily sample of 10, and plot the resultant sampling distribution of the mean.

```python
# Using a foor loop to perform this sampling of 10 for each day of a year
#[np.mean(townsfolk_sampler(10)) for x in range(365)]
trial_10 = []
for x in range(365):
    trial_10.append(np.mean(townsfolk_sampler(10)))
```

```python
seed(47)
# plotting the sampling distribution of the mean
_ = plt.hist(trial_10, bins = 10)
_ = plt.xlabel('height (cm)')
_ = plt.ylabel('number of people')
_ = plt.title('Distribution of heights of daily sample of 10 for a year')
_ = plt.axvline(172, color='r')
_ = plt.axvline(172+5, color='r', linestyle='--')
_ = plt.axvline(172-5, color='r', linestyle='--')
_ = plt.axvline(172+10, color='r', linestyle='-.')
_ = plt.axvline(172-10, color='r', linestyle='-.')

print(len(trial_10))
print(np.mean(trial_10))
print(np.median(trial_10))
```

The above is the distribution of the means of samples of size 10 taken from our population. The Central Limit Theorem tells us the expected mean of this distribution will be equal to the population mean, and standard deviation will be  𝜎/𝑛⎯⎯√ , which, in this case, should be approximately 1.58.

Q: Verify the above results from the CLT.

```python
np.std(trial_10, ddof=1)
```

    1.578160835870796

Remember, in this instance, we knew our population parameters, that the average height really is 172 cm and the standard deviation is 5 cm, and we see some of our daily estimates of the population mean were as low as around 168 and some as high as 176.

Q: Repeat the above year's worth of samples but for a sample size of 50 (perhaps you had a bigger budget for conducting surveys that year!) Would you expect your distribution of sample means to be wider (more variable) or narrower (more consistent)? Compare your resultant summary statistics to those predicted by the CLT.

```python
seed(47)
trial_50 = []
for x in range(365):
    trial_50.append(np.mean(townsfolk_sampler(50)))
```

```python
_ = plt.hist(trial_50, bins = 50, histtype='stepfilled', alpha=0.8)
_ = plt.xlabel('height (cm)')
_ = plt.ylabel('number of people')
_ = plt.title('Distribution of heights of daily sample of 50 for a year')
_ = plt.axvline(172, color='r')
_ = plt.axvline(172+5, color='r', linestyle='--')
_ = plt.axvline(172-5, color='r', linestyle='--')
_ = plt.axvline(172+10, color='r', linestyle='-.')
_ = plt.axvline(172-10, color='r', linestyle='-.')

print(len(trial_50))
print(np.mean(trial_50))
print(np.median(trial_50))
print(np.std(trial_50,ddof = 1))
```

What we've seen so far, then, is that we can estimate population parameters from a sample from the population, and that samples have their own distributions. Furthermore, the larger the sample size, the narrower are those sampling distributions.

### Let's now start from the position of knowing nothing about the heights of people in our town.
* Use our favorite random seed of 47, to randomly sample the heights of 50 townsfolk
* Estimate the population mean using np.mean
* Estimate the population standard deviation using np.std (remember which denominator to use!)
* Calculate the (95%) [margin of error](https://www.statisticshowto.datasciencecentral.com/probability-and-statistics/hypothesis-testing/margin-of-error/#WhatMofE) (use the exact critial z value to 2 decimal places - [look this up](https://www.statisticshowto.datasciencecentral.com/probability-and-statistics/find-critical-values/) or use norm.ppf())
* Calculate the 95% Confidence Interval of the mean
* Does this interval include the true population mean?

```python
seed(47)
second_trial_50 = townsfolk_sampler(50)
```

```python
np.mean(second_trial_50)
```

    172.7815108576788

```python
# Using ddof = 1 because this is a sample not population
np.std(second_trial_50, ddof=1)
```

    4.195424364433547

```python
critical_val = (norm.ppf(0.95))
# Standard Error of the Mean (a.k.a. the standard deviation of the sampling distribution of the sample mean!
se = (np.std(second_trial_50, ddof=1)) / (np.sqrt(50))
moe = critical_val * se # Margin of Error
print(moe)
```

    0.9759288364989565

```python
lower=np.mean(second_trial_50) - moe
upper=np.mean(second_trial_50) + moe
print('lower: ' + str(lower) + '\n' + 'Upper: ' + str(upper))
```

    lower: 171.80558202117984
    Upper: 173.75743969417775

__Q:__ Above we calculated the confidence interval using the critical z value. What is the problem with this? What requirement, or requirements, are we (strictly) failing?

__A:__ Because we don't know anything about our population, the dataset has an unknown population standard deviation. I think using a t-score instead of z-score would be better.

__Q__: Calculate the 95% confidence interval for the mean using the _t_ distribution. Is this wider or narrower than that based on the normal distribution above? If you're unsure, you may find this [resource](https://www.statisticshowto.datasciencecentral.com/probability-and-statistics/confidence-interval/) useful. For calculating the critical value, remember how you could calculate this for the normal distribution using norm.ppf().

```python
#Subtract 1 from your sample size to find the degrees of freedom (df).
dof = 50 - 1
t = t.ppf(0.95, (dof))
t
```

    1.6765508919142629

```python
# Margin of error for t-score
moe_t = t * se
```

```python
lower=np.mean(second_trial_50) - moe_t
upper=np.mean(second_trial_50) + moe_t
print('lower: ' + str(lower) + '\n' + 'Upper: ' + str(upper))
```

    lower: 171.78677531740482
    Upper: 173.77624639795278

__A:__ Wider than the normal distribution since t-distribution curves are wider and bigger at the tails compared to normal distribution curves

```python

```

Python Statistics/Mini Projects/inferential_statistics_frequentist_mini-projects6.28.19/Mini Projects - Simplified.ipynb


# Basis for grayscale images

## Introduction

Consider the set of real-valued matrices of size $M\times N$; we can turn this into a vector space by defining addition and scalar multiplication in the usual way:

\begin{align}
\mathbf{A} + \mathbf{B} &=
    \left[
        \begin{array}{ccc}
            a_{0,0} & \dots & a_{0,N-1} \\
            \vdots & & \vdots \\
            a_{M-1,0} & \dots & b_{M-1,N-1}
        \end{array}
    \right]
    +
    \left[
        \begin{array}{ccc}
            b_{0,0} & \dots & b_{0,N-1} \\
            \vdots & & \vdots \\
            b_{M-1,0} & \dots & b_{M-1,N-1}
        \end{array}
    \right]
    \\
    &=
    \left[
        \begin{array}{ccc}
            a_{0,0}+b_{0,0} & \dots & a_{0,N-1}+b_{0,N-1} \\
            \vdots & & \vdots \\
            a_{M-1,0}+b_{M-1,0} & \dots & a_{M-1,N-1}+b_{M-1,N-1}
        \end{array}
    \right]
    \\ \\ \\
\beta\mathbf{A} &=
    \left[
        \begin{array}{ccc}
            \beta a_{0,0} & \dots & \beta a_{0,N-1} \\
            \vdots & & \vdots \\
            \beta a_{M-1,0} & \dots & \beta a_{M-1,N-1}
        \end{array}
    \right]
\end{align}

As a matter of fact, the space of real-valued $M\times N$ matrices is completely equivalent to $\mathbb{R}^{MN}$ and we can always "unroll" a matrix into a vector. Assume we proceed column by column; then the matrix becomes

$$
    \mathbf{a} = \mathbf{A}[:] = [
        \begin{array}{ccccccc}
            a_{0,0} & \dots & a_{M-1,0} & a_{0,1} & \dots & a_{M-1,1} & \ldots & a_{0, N-1} & \dots & a_{M-1,N-1}
        \end{array}]^T
$$

Although the matrix and vector forms represent exactly the same data, the matrix form allows us to display the data in the form of an image. Assume each value in the matrix is a grayscale intensity, where zero is black and 255 is white; for example we can create a checkerboard pattern of any size with the following function:

```python
# usual python bookkeeping...
%pylab inline
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import IPython
from IPython.display import Image
import math
from __future__ import print_function

# ensure all images will be grayscale
gray();
```

    Populating the interactive namespace from numpy and matplotlib

    <Figure size 432x288 with 0 Axes>

#### (?1) `gray()  # ensure all images will be grayscale`? What?

#### (R1)

```python
gray.__module__
```

    'matplotlib.pyplot'

```python
help(gray)
```

    Help on function gray in module matplotlib.pyplot:

    gray()
        Set the colormap to "gray".

        This changes the default colormap as well as the colormap of the current
        image if there is one. See ``help(colormaps)`` for more information.

#### (!1) Worth digging deep: `colormaps`

```python
help(colormaps)
```

    Help on function colormaps in module matplotlib.pyplot:

    colormaps()
        Matplotlib provides a number of colormaps, and others can be added using
        :func:`~matplotlib.cm.register_cmap`.  This function documents the built-in
        colormaps, and will also return a list of all registered colormaps if
        called.

        You can set the colormap for an image, pcolor, scatter, etc,
        using a keyword argument::

          imshow(X, cmap=cm.hot)

        or using the :func:`set_cmap` function::

          imshow(X)
          pyplot.set_cmap('hot')
          pyplot.set_cmap('jet')

        In interactive mode, :func:`set_cmap` will update the colormap post-hoc,
        allowing you to see which one works best for your data.

        All built-in colormaps can be reversed by appending ``_r``: For instance,
        ``gray_r`` is the reverse of ``gray``.

        There are several common color schemes used in visualization:

        Sequential schemes
          for unipolar data that progresses from low to high
        Diverging schemes
          for bipolar data that emphasizes positive or negative deviations from a
          central value
        Cyclic schemes
          for plotting values that wrap around at the endpoints, such as phase
          angle, wind direction, or time of day
        Qualitative schemes
          for nominal data that has no inherent ordering, where color is used
          only to distinguish categories

        Matplotlib ships with 4 perceptually uniform color maps which are
        the recommended color maps for sequential data:

          =========   ===================================================
          Colormap    Description
          =========   ===================================================
          inferno     perceptually uniform shades of black-red-yellow
          magma       perceptually uniform shades of black-red-white
          plasma      perceptually uniform shades of blue-red-yellow
          viridis     perceptually uniform shades of blue-green-yellow
          =========   ===================================================

        The following colormaps are based on the `ColorBrewer
        <https://colorbrewer2.org>`_ color specifications and designs developed by
        Cynthia Brewer:

        ColorBrewer Diverging (luminance is highest at the midpoint, and
        decreases towards differently-colored endpoints):

          ========  ===================================
          Colormap  Description
          ========  ===================================
          BrBG      brown, white, blue-green
          PiYG      pink, white, yellow-green
          PRGn      purple, white, green
          PuOr      orange, white, purple
          RdBu      red, white, blue
          RdGy      red, white, gray
          RdYlBu    red, yellow, blue
          RdYlGn    red, yellow, green
          Spectral  red, orange, yellow, green, blue
          ========  ===================================

        ColorBrewer Sequential (luminance decreases monotonically):

          ========  ====================================
          Colormap  Description
          ========  ====================================
          Blues     white to dark blue
          BuGn      white, light blue, dark green
          BuPu      white, light blue, dark purple
          GnBu      white, light green, dark blue
          Greens    white to dark green
          Greys     white to black (not linear)
          Oranges   white, orange, dark brown
          OrRd      white, orange, dark red
          PuBu      white, light purple, dark blue
          PuBuGn    white, light purple, dark green
          PuRd      white, light purple, dark red
          Purples   white to dark purple
          RdPu      white, pink, dark purple
          Reds      white to dark red
          YlGn      light yellow, dark green
          YlGnBu    light yellow, light green, dark blue
          YlOrBr    light yellow, orange, dark brown
          YlOrRd    light yellow, orange, dark red
          ========  ====================================

        ColorBrewer Qualitative:

        (For plotting nominal data, `.ListedColormap` is used,
        not `.LinearSegmentedColormap`.  Different sets of colors are
        recommended for different numbers of categories.)

        * Accent
        * Dark2
        * Paired
        * Pastel1
        * Pastel2
        * Set1
        * Set2
        * Set3

        A set of colormaps derived from those of the same name provided
        with Matlab are also included:

          =========   =======================================================
          Colormap    Description
          =========   =======================================================
          autumn      sequential linearly-increasing shades of red-orange-yellow
          bone        sequential increasing black-white color map with
                      a tinge of blue, to emulate X-ray film
          cool        linearly-decreasing shades of cyan-magenta
          copper      sequential increasing shades of black-copper
          flag        repetitive red-white-blue-black pattern (not cyclic at
                      endpoints)
          gray        sequential linearly-increasing black-to-white
                      grayscale
          hot         sequential black-red-yellow-white, to emulate blackbody
                      radiation from an object at increasing temperatures
          jet         a spectral map with dark endpoints, blue-cyan-yellow-red;
                      based on a fluid-jet simulation by NCSA [#]_
          pink        sequential increasing pastel black-pink-white, meant
                      for sepia tone colorization of photographs
          prism       repetitive red-yellow-green-blue-purple-...-green pattern
                      (not cyclic at endpoints)
          spring      linearly-increasing shades of magenta-yellow
          summer      sequential linearly-increasing shades of green-yellow
          winter      linearly-increasing shades of blue-green
          =========   =======================================================

        A set of palettes from the `Yorick scientific visualisation
        package <https://dhmunro.github.io/yorick-doc/>`_, an evolution of
        the GIST package, both by David H. Munro are included:

          ============  =======================================================
          Colormap      Description
          ============  =======================================================
          gist_earth    mapmaker's colors from dark blue deep ocean to green
                        lowlands to brown highlands to white mountains
          gist_heat     sequential increasing black-red-orange-white, to emulate
                        blackbody radiation from an iron bar as it grows hotter
          gist_ncar     pseudo-spectral black-blue-green-yellow-red-purple-white
                        colormap from National Center for Atmospheric
                        Research [#]_
          gist_rainbow  runs through the colors in spectral order from red to
                        violet at full saturation (like *hsv* but not cyclic)
          gist_stern    "Stern special" color table from Interactive Data
                        Language software
          ============  =======================================================

        A set of cyclic color maps:

          ================  =================================================
          Colormap          Description
          ================  =================================================
          hsv               red-yellow-green-cyan-blue-magenta-red, formed by
                            changing the hue component in the HSV color space
          twilight          perceptually uniform shades of
                            white-blue-black-red-white
          twilight_shifted  perceptually uniform shades of
                            black-blue-white-red-black
          ================  =================================================

        Other miscellaneous schemes:

          ============= =======================================================
          Colormap      Description
          ============= =======================================================
          afmhot        sequential black-orange-yellow-white blackbody
                        spectrum, commonly used in atomic force microscopy
          brg           blue-red-green
          bwr           diverging blue-white-red
          coolwarm      diverging blue-gray-red, meant to avoid issues with 3D
                        shading, color blindness, and ordering of colors [#]_
          CMRmap        "Default colormaps on color images often reproduce to
                        confusing grayscale images. The proposed colormap
                        maintains an aesthetically pleasing color image that
                        automatically reproduces to a monotonic grayscale with
                        discrete, quantifiable saturation levels." [#]_
          cubehelix     Unlike most other color schemes cubehelix was designed
                        by D.A. Green to be monotonically increasing in terms
                        of perceived brightness. Also, when printed on a black
                        and white postscript printer, the scheme results in a
                        greyscale with monotonically increasing brightness.
                        This color scheme is named cubehelix because the (r, g, b)
                        values produced can be visualised as a squashed helix
                        around the diagonal in the (r, g, b) color cube.
          gnuplot       gnuplot's traditional pm3d scheme
                        (black-blue-red-yellow)
          gnuplot2      sequential color printable as gray
                        (black-blue-violet-yellow-white)
          ocean         green-blue-white
          rainbow       spectral purple-blue-green-yellow-orange-red colormap
                        with diverging luminance
          seismic       diverging blue-white-red
          nipy_spectral black-purple-blue-green-yellow-red-white spectrum,
                        originally from the Neuroimaging in Python project
          terrain       mapmaker's colors, blue-green-yellow-brown-white,
                        originally from IGOR Pro
          turbo         Spectral map (purple-blue-green-yellow-orange-red) with
                        a bright center and darker endpoints. A smoother
                        alternative to jet.
          ============= =======================================================

        The following colormaps are redundant and may be removed in future
        versions.  It's recommended to use the names in the descriptions
        instead, which produce identical output:

          =========  =======================================================
          Colormap   Description
          =========  =======================================================
          gist_gray  identical to *gray*
          gist_yarg  identical to *gray_r*
          binary     identical to *gray_r*
          =========  =======================================================

        .. rubric:: Footnotes

        .. [#] Rainbow colormaps, ``jet`` in particular, are considered a poor
          choice for scientific visualization by many researchers: `Rainbow Color
          Map (Still) Considered Harmful
          <https://ieeexplore.ieee.org/document/4118486/?arnumber=4118486>`_

        .. [#] Resembles "BkBlAqGrYeOrReViWh200" from NCAR Command
          Language. See `Color Table Gallery
          <https://www.ncl.ucar.edu/Document/Graphics/color_table_gallery.shtml>`_

        .. [#] See `Diverging Color Maps for Scientific Visualization
          <http://www.kennethmoreland.com/color-maps/>`_ by Kenneth Moreland.

        .. [#] See `A Color Map for Effective Black-and-White Rendering of
          Color-Scale Images
          <https://www.mathworks.com/matlabcentral/fileexchange/2662-cmrmap-m>`_
          by Carey Rappaport

```python
# let's create a checkerboard pattern
SIZE = 4
img = np.zeros((SIZE, SIZE))
for n in range(0, SIZE):
    for m in range(0, SIZE):
        if (n & 0x1) ^ (m & 0x1):
            ## recall that ^ is the exclusive or
            #img[n, m] = 255
            img[m, n] = 255

# now display the matrix as an image
plt.matshow(img);
```

```python
SIZE = 4
img = np.zeros((SIZE, SIZE))
img.dtype  # float64, better uint8 for images
```

    dtype('float64')

```python
np.uint8
```

    numpy.uint8

`m & 0x1` and `n & 0x1` tests the parity: if `m` is odd, then `m & 0x1` outputs `1` (or `True`); if `m` even,
output `0`. Using XOR after that guarantees that only when exactly one of `m` and `n` is odd will the case
be colored white. This implies that every time one crosses from one case to any of its neighboring cases,
the color changes

Note how the axes of `plt.matshow()` are arranged diff from that of `plt.imshow()`
help(plt.matshow)

```python
plt.imshow(img);
```

Given the equivalence between the space of $M\times N$ matrices and $\mathbb{R}^{MN}$ we can easily define the inner product between two matrices in the usual way:

$$
\langle \mathbf{A}, \mathbf{B} \rangle = \sum_{m=0}^{M-1} \sum_{n=0}^{N-1} a_{m,n} b_{m, n}
$$

(where we have neglected the conjugation since we'll only deal with real-valued matrices); in other words, we can take the inner product between two matrices as the standard inner product of their unrolled versions. The inner product allows us to define orthogonality between images and this is rather useful since we're going to explore a couple of bases for this space.

## Actual images

Conveniently, using IPython, we can read images from disk in any given format and convert them to numpy arrays; let's load and display for instance a JPEG image:
img = np.array(plt.imread('cameraman.jpg'), dtype=int)
plt.matshow(img);---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-9-37d88f4789f6> in <module>()
----> 1 img = np.array(plt.imread('cameraman.jpg'), dtype=int)
      2 plt.matshow(img);

/home/phunc20/.virtualenvs/dsp-py2.7/lib/python2.7/site-packages/matplotlib/pyplot.pyc in imread(*args, **kwargs)
   2371 @docstring.copy_dedent(_imread)
   2372 def imread(*args, **kwargs):
-> 2373     return _imread(*args, **kwargs)
   2374
   2375

/home/phunc20/.virtualenvs/dsp-py2.7/lib/python2.7/site-packages/matplotlib/image.pyc in imread(fname, format)
   1351             raise ValueError('Only know how to handle extensions: %s; '
   1352                              'with Pillow installed matplotlib can handle '
-> 1353                              'more images' % list(handlers))
   1354         return im
   1355

ValueError: Only know how to handle extensions: [u'png']; with Pillow installed matplotlib can handle more images!pip install pillow

```python
img = np.array(plt.imread('cameraman.jpg'), dtype=int)
plt.matshow(img);
```

The image is a $64\times 64$ low-resolution version of the famous "cameraman" test picture. Out of curiosity, we can look at the first column of this image, which is is a $64×1$ vector:

```python
img[:,0]
```

    array([156, 157, 157, 152, 154, 155, 151, 157, 152, 155, 158, 159, 159,
           160, 160, 161, 155, 160, 161, 161, 164, 162, 160, 162, 158, 160,
           158, 157, 160, 160, 159, 158, 163, 162, 162, 157, 160, 114, 114,
           103,  88,  62, 109,  82, 108, 128, 138, 140, 136, 128, 122, 137,
           147, 114, 114, 144, 112, 115, 117, 131, 112, 141,  99,  97])

The values are integers between zero and 255, meaning that each pixel is encoded over 8 bits (or 256 gray levels).

## The canonical basis

The canonical basis for any matrix space $\mathbb{R}^{M\times N}$ is the set of "delta" matrices where only one element equals to one while all the others are 0. Let's call them $\mathbf{E}_n$ with $0 \leq n < MN$. Here is a function to create the canonical basis vector given its index:

```python
def canonical(n, M=5, N=10):
    e = np.zeros((M, N))
    #e[(n % M), int(n / M)] = 1
    e[n % M, n // M] = 1
    return e
```

```python
!python --version
```

    Python 3.8.2

```python
10 / 3, 10 // 3, int(10/3)
```

    (3.3333333333333335, 3, 3)

Here are some basis vectors: look for the position of white pixel, which differentiates them and note that we enumerate pixels column-wise:

```python
plt.matshow(canonical(0));
plt.matshow(canonical(1));
plt.matshow(canonical(49));
```

Note how diff `matshow()` is from `imshow()`:
> In a jupyter cell, successive `matshow()`'s can draw any number of images, while `imshow()` will only do it for the last one

```python
plt.imshow(canonical(0));
plt.imshow(canonical(1));
plt.imshow(canonical(49));
```

##### Stopped here (2020/11/19 12h25)

## Transmitting images

Suppose we want to transmit the "cameraman" image over a communication channel. The intuitive way to do so is to send the  pixel values one by one, which corresponds to sending the coefficients of the decomposition of the image over the canonical basis. So far, nothing complicated: to send the cameraman image, for instance, we will send $64\times 64 = 4096$ coefficients in a row.

Now suppose that a communication failure takes place after the first half of the pixels have been sent. The received data will allow us to display an approximation of the original image only. If we replace the missing data with zeros, here is what we would see, which is not very pretty:

```python
# unrolling of the image for transmission (we go column by column, hence "F")
tx_img = np.ravel(img, "F")

# oops, we lose half the data
tx_img[int(len(tx_img)/2):] = 0

# rebuild matrix
rx_img = np.reshape(tx_img, (64, 64), "F")
plt.matshow(rx_img);
```
help(np.ravel)
Can we come up with a trasmission scheme that is more robust in the face of channel loss? Interestingly, the answer is yes, and it involves a different, more versatile basis for the space of images. What we will do is the following:

* describe the Haar basis, a new basis for the image space
* project the image in the new basis
* transmit the projection coefficients
* rebuild the image using the basis vectors

We know a few things: if we choose an orthonormal basis, the analysis and synthesis formulas will be super easy (a simple inner product and a scalar multiplication respectively). The trick is to find a basis that will be robust to the loss of some coefficients.

One such basis is the **Haar basis**. We cannot go into too many details in this notebook but, for the curious, a good starting point is [here](https://chengtsolin.wordpress.com/2015/04/15/real-time-2d-discrete-wavelet-transform-using-opengl-compute-shader/). (An even better starting point is [these two papers](http://grail.cs.washington.edu/projects/wavelets/article/).) Mathematical formulas aside, the Haar basis works by encoding the information in a *hierarchical* way: the first basis vectors encode the broad information and the higher coefficients encode the detail. Let's have a look.

First of all, to keep things simple, we will remain in the space of square matrices whose size is a power of two. The code to generate the Haar basis matrices is the following: first we generate a 1D Haar vector and then we obtain the basis matrices by taking the outer product of all possible 1D vectors (don't worry if it's not clear, the results are what's important):

```python
def haar1D(n, SIZE):
    # check power of two
    if math.floor(math.log(SIZE) / math.log(2)) != math.log(SIZE) / math.log(2):
        print("Haar defined only for lengths that are a power of two")
        return None
    if n >= SIZE or n < 0:
        print("invalid Haar index")
        return None

    # zero basis vector
    if n == 0:
        return np.ones(SIZE)

    # express n >= 1 as 2^p + q with p as large as possible;
    # then k = SIZE/2^p is the length of the support
    # and s = qk is the shift
    p = math.floor(math.log(n) / math.log(2))
    pp = int(pow(2, p))
    k = SIZE / pp
    s = (n - pp) * k

    h = np.zeros(SIZE)
    h[int(s):int(s+k/2)] = 1
    h[int(s+k/2):int(s+k)] = -1
    # these are not normalized
    return h

def haar2D(n, SIZE=8):
    # get horizontal and vertical indices
    hr = haar1D(n % SIZE, SIZE)
    hv = haar1D(int(n / SIZE), SIZE)
    # 2D Haar basis matrix is separable, so we can
    # just take the column-row product
    H = np.outer(hr, hv)
    # np.outer() is just column vector times row vector,
    # the 1st arg being the col vec, the 2nd the row vec.
    H = H / math.sqrt(np.sum(H * H))
    # the previous line just divides H by its Frobenius norm
    # so that the returned value of haar2D() has norm 1.
    return H
```
help(np.outer)
First of all, let's look at a few basis matrices; note that the matrices have **both positive and negative values**, so that the value of **zero** will be represented as **gray**:

```python
plt.matshow?
```

```python
plt.matshow(haar2D(0));
plt.matshow(haar2D(1));
plt.matshow(haar2D(10));
plt.matshow(haar2D(63));
```

```python
np.unique(haar2D(0))
```

    array([0.125])

```python
np.unique(haar2D(1))
```

    array([-0.125,  0.125])

```python
np.unique(haar2D(63))
```

    array([-0.5,  0. ,  0.5])

```python
white = np.unique(haar2D(0))[0]
black = -white
plt.matshow(haar2D(0), vmax=white, vmin=black);
```

We can notice two key properties

* each basis matrix has positive and negative values in some symmetric pattern: this means that the basis matrix will implicitly compute the difference between image areas
* low-index basis matrices take differences between large areas, while high-index ones take differences in smaller **localized** areas of the image

We can immediately verify that the Haar matrices are orthogonal:

```python
# let's use an 8x8 space; there will be 64 basis vectors
# compute all possible inner product and only print the nonzero results
for m in range(0,64):
    for n in range(0,64):
        r = np.sum(haar2D(m, 8) * haar2D(n, 8))
        if r != 0:
            print("[%dx%d -> %f] " % (m, n, r), end="")
```

    [0x0 -> 1.000000] [1x1 -> 1.000000] [2x2 -> 1.000000] [3x3 -> 1.000000] [4x4 -> 1.000000] [5x5 -> 1.000000] [6x6 -> 1.000000] [7x7 -> 1.000000] [8x8 -> 1.000000] [9x9 -> 1.000000] [10x10 -> 1.000000] [11x11 -> 1.000000] [12x12 -> 1.000000] [13x13 -> 1.000000] [14x14 -> 1.000000] [15x15 -> 1.000000] [16x16 -> 1.000000] [16x17 -> -0.000000] [17x16 -> -0.000000] [17x17 -> 1.000000] [18x18 -> 1.000000] [19x19 -> 1.000000] [20x20 -> 1.000000] [21x21 -> 1.000000] [22x22 -> 1.000000] [23x23 -> 1.000000] [24x24 -> 1.000000] [24x25 -> -0.000000] [25x24 -> -0.000000] [25x25 -> 1.000000] [26x26 -> 1.000000] [27x27 -> 1.000000] [28x28 -> 1.000000] [29x29 -> 1.000000] [30x30 -> 1.000000] [31x31 -> 1.000000] [32x32 -> 1.000000] [33x33 -> 1.000000] [34x34 -> 1.000000] [35x35 -> 1.000000] [36x36 -> 1.000000] [37x37 -> 1.000000] [38x38 -> 1.000000] [39x39 -> 1.000000] [40x40 -> 1.000000] [41x41 -> 1.000000] [42x42 -> 1.000000] [43x43 -> 1.000000] [44x44 -> 1.000000] [45x45 -> 1.000000] [46x46 -> 1.000000] [47x47 -> 1.000000] [48x48 -> 1.000000] [49x49 -> 1.000000] [50x50 -> 1.000000] [51x51 -> 1.000000] [52x52 -> 1.000000] [53x53 -> 1.000000] [54x54 -> 1.000000] [55x55 -> 1.000000] [56x56 -> 1.000000] [57x57 -> 1.000000] [58x58 -> 1.000000] [59x59 -> 1.000000] [60x60 -> 1.000000] [61x61 -> 1.000000] [62x62 -> 1.000000] [63x63 -> 1.000000]

OK! Everything's fine. Now let's transmit the "cameraman" image: first, let's verify that it works

```python
# project the image onto the Haar basis, obtaining a vector of 4096 coefficients
# this is simply the analysis formula for the vector space with an orthogonal basis
tx_img = np.zeros(64*64)
for k in range(0, (64*64)):
    tx_img[k] = np.sum(img * haar2D(k, 64))

# now rebuild the image with the synthesis formula; since the basis is orthonormal
# we just need to scale the basis matrices by the projection coefficients
rx_img = np.zeros((64, 64))
for k in range(0, (64*64)):
    rx_img += tx_img[k] * haar2D(k, 64)

plt.matshow(rx_img);
```

```python
np.linalg.norm(img - rx_img, inf)
```

    2.112088282046898e-12

help(np.linalg.norm)
Cool, it works! Now let's see what happens if we lose the second half of the coefficients:

```python
# oops, we lose half the data
lossy_img = np.copy(tx_img);
lossy_img[int(len(tx_img)/2):] = 0

# rebuild matrix
rx_img = np.zeros((64, 64))
for k in range(0, (64*64)):
    rx_img += lossy_img[k] * haar2D(k, 64)

plt.matshow(rx_img);
```

That's quite remarkable, no? We've lost the same amount of information as before but the image is still acceptable. This is because we lost the coefficients associated to the fine details of the image but we retained the "broad strokes" encoded by the first half.

Note that if we lose the first half of the coefficients, the result would look remarkably different:

```python
lossy_img = np.copy(tx_img);
lossy_img[0:int(len(tx_img)/2)] = 0

rx_img = np.zeros((64, 64))
for k in range(0, (64*64)):
    rx_img += lossy_img[k] * haar2D(k, 64)

plt.matshow(rx_img);
```

In fact, schemes like this one are used in *progressive encoding*: send the most important information first and add details if the channel permits it. You may have experienced this while browsing the internet over a slow connection.

All in all, a great application of a change of basis!

## A few of my own questions

**(?)** About `# check power of two` of `haar1D()`

```python
help(math.log)
```

    Help on built-in function log in module math:

    log(...)
        log(x[, base])

        Return the logarithm of x to the given base.
        If the base not specified, returns the natural logarithm (base e) of x.

```python
math.log(2)
```

    0.6931471805599453

```python
math.log(2, 2)
```

    1.0

```python
math.log(2, 10)
```

    0.30102999566398114

The author is just checking whether
$
\log_{2} \texttt{SIZE} = \frac{\ln \texttt{SIZE}}{\ln 2}
$
is an integer.

**N.B.** Unlike `math.log`, numpy has
- `np.log`: Natural logarithm
- `np.log2`: base 2
- `np.log10`: base 10
- `np.log1p`: log(1+p)

**(?)** What is `n` in `haar2D()`?
Must `n` be bounded?

**(R)**
From reading the code, it seems that if `SIZE = k`, then `n = 0, 1, 2, ..., k**2 -1`, like the above examples
- when `SIZE=8`, `n = 0, 1, ..., 63`
- when `SIZE=64`, `n = 0, 1, ..., 64**2 - 1`

The `n` in `haar2D()` has to do with the `n` in `haar1D()`. To better understand how the function is written that way, readers would better have read [one of the papers](http://grail.cs.washington.edu/projects/wavelets/article/wavelet1.pdf) mentioned above. Briefly speaking, `Haar1D(n , SIZE)` will return a basis for the vector space $V^j$, where with `SIZE`$ = 2^j$. For example, with the box basis and the Haar wavelets described in the paper,

$$
\forall\, f \in V^3,\; \text{we have}\\
f =
c_{0}^{0} \phi_{0}^{0}
+ d_{0}^{0} \psi_{0}^{0}
+ \left(d_{0}^{1} \psi_{0}^{1} + d_{1}^{1} \psi_{1}^{1}\right)
+ \left(d_{0}^{2} \psi_{0}^{2} + d_{1}^{2} \psi_{1}^{2} + d_{2}^{2} \psi_{2}^{2} + d_{3}^{2} \psi_{3}^{2}\right).
$$
<br>
What `haar1D(n, SIZE)` does is that it returns those $\phi_{s}^{t}$ and $\psi_{s}^{t}$.<br> As you can see from the $V^3$ example, there should be $2^3$ such basis vectors/functions; in general, there will be `SIZE`$= 2^j$ basis vectors and that's why the subscript (here in the `haar1D` function `n`) runs from `0` to `SIZE-1`.

**(?)** Is it correct for `haar2D` to normalize by Frobenius norm?

**(R)**

## Ref.
- [http://grail.cs.washington.edu/projects/wavelets/](http://grail.cs.washington.edu/projects/wavelets/)
    - [http://grail.cs.washington.edu/Research/](http://grail.cs.washington.edu/Research/)
- **exercise3.4 p.57** of the textbook written by the same authors of the course also talks about Haar basis.

```python
type(pow(2,3)), type(pow(2,3.0))
```

    (int, float)

```python
pow(2,3), pow(2,3.0)
```

    (8, 8.0)

```python
2**3, 2**3.0
```

    (8, 8.0)

```python

```