### Gradient Descent

Jay Urbain, PhD

Credits:
- https://gist.github.com/sagarmainkar

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

```

```python
plt.style.use(['ggplot'])
```

# Create Data

 Generate some data with:
\begin{equation} \theta_0= 4 \end{equation}
\begin{equation} \theta_1= 3 \end{equation}

Add some Gaussian noise to the data

```python
X = 2 * np.random.rand(100,1)
y = 4 +3 * X+np.random.randn(100,1)
```

Let's plot our data to check the relation between X and Y

```python

plt.plot(X,y,'b.')
plt.xlabel("$x$", fontsize=18)
plt.ylabel("$y$", rotation=0, fontsize=18)
_ =plt.axis([0,2,0,15])
```

#  Analytical way of Linear Regression

```python
X_b = np.c_[np.ones((100,1)),X]
theta_best = np.linalg.inv(X_b.T.dot(X_b)).dot(X_b.T).dot(y)
print(theta_best)
```

    [[4.19057743]
     [2.98123237]]

This is close to our real thetas 4 and 3. It cannot be accurate due to the noise I have introduced in data

```python
X_new = np.array([[0],[2]])
X_new_b = np.c_[np.ones((2,1)),X_new]
y_predict = X_new_b.dot(theta_best)
y_predict
```

    array([[ 4.19057743],
           [10.15304217]])

Let's plot prediction line with calculated:theta

```python
plt.plot(X_new,y_predict,'r-')
plt.plot(X,y,'b.')
plt.xlabel("$x_1$", fontsize=18)
plt.ylabel("$y$", rotation=0, fontsize=18)
plt.axis([0,2,0,15])

```

# Gradient Descent

## Cost Function & Gradients

 The equation for calculating cost function and gradients are as shown below. Please note the cost function is for Linear regression. For other algorithms the cost function will be different and the gradients would have  to be derived from the cost functions

Cost
\begin{equation}
J(\theta) = 1/2m \sum_{i=1}^{m} (h(\theta)^{(i)} - y^{(i)})^2
\end{equation}

Gradient

\begin{equation}
\frac{\partial J(\theta)}{\partial \theta_j} = 1/m\sum_{i=1}^{m}(h(\theta^{(i)} - y^{(i)}).X_j^{(i)}
\end{equation}

Gradients
\begin{equation}
\theta_0: = \theta_0 -\alpha . (1/m .\sum_{i=1}^{m}(h(\theta^{(i)} - y^{(i)}).X_0^{(i)})
\end{equation}
\begin{equation}
\theta_1: = \theta_1 -\alpha . (1/m .\sum_{i=1}^{m}(h(\theta^{(i)} - y^{(i)}).X_1^{(i)})
\end{equation}
\begin{equation}
\theta_2: = \theta_2 -\alpha . (1/m .\sum_{i=1}^{m}(h(\theta^{(i)} - y^{(i)}).X_2^{(i)})
\end{equation}

\begin{equation}
\theta_j: = \theta_j -\alpha . (1/m .\sum_{i=1}^{m}(h(\theta^{(i)} - y^{(i)}).X_0^{(i)})
\end{equation}

```python

def  cal_cost(theta,X,y):
    '''

    Calculates the cost for given X and Y. The following shows and example of a single dimensional X
    theta = Vector of thetas
    X     = Row of X's np.zeros((2,j))
    y     = Actual y's np.zeros((2,1))

    where:
        j is the no of features
    '''

    m = len(y)

    predictions = X.dot(theta)
    cost = (1/2*m) * np.sum(np.square(predictions-y))
    return cost

```

```python
def gradient_descent(X,y,theta,learning_rate=0.01,iterations=100):
    '''
    X    = Matrix of X with added bias units
    y    = Vector of Y
    theta=Vector of thetas np.random.randn(j,1)
    learning_rate
    iterations = no of iterations

    Returns the final theta vector and array of cost history over no of iterations
    '''
    m = len(y)
    cost_history = np.zeros(iterations)
    theta_history = np.zeros((iterations,2))
    for it in range(iterations):

        prediction = np.dot(X,theta)

        theta = theta -(1/m)*learning_rate*( X.T.dot((prediction - y)))
        theta_history[it,:] =theta.T
        cost_history[it]  = cal_cost(theta,X,y)

    return theta, cost_history, theta_history

```

 Let's start with 1000 iterations and a learning rate of 0.01. Start with theta from a Gaussian distribution

```python
lr =0.01
n_iter = 1000

theta = np.random.randn(2,1)

X_b = np.c_[np.ones((len(X),1)),X]
theta,cost_history,theta_history = gradient_descent(X_b,y,theta,lr,n_iter)

print('Theta0:          {:0.3f},\nTheta1:          {:0.3f}'.format(theta[0][0],theta[1][0]))
print('Final cost/MSE:  {:0.3f}'.format(cost_history[-1]))
```

    Theta0:          4.063,
    Theta1:          3.088
    Final cost/MSE:  5864.178

 Let's plot the cost history over iterations

```python
fig,ax = plt.subplots(figsize=(12,8))

ax.set_ylabel('J(Theta)')
ax.set_xlabel('Iterations')
_=ax.plot(range(n_iter),cost_history,'b.')
```

 After around 150 iterations the cost is flat so the remaining iterations  are not needed or will not result in any further optimization. Let us zoom in till iteration 200 and see the curve

```python

fig,ax = plt.subplots(figsize=(10,8))
_=ax.plot(range(200),cost_history[:200],'b.')
```

It is worth while to note that the cost drops faster initially and then the gain in cost reduction is not as much

### It would be great to see the effect of different learning rates and iterations together

### Let us  build a function which can show the effects together and also show how gradient decent actually is working

```python

def plot_GD(n_iter,lr,ax,ax1=None):
     """
     n_iter = no of iterations
     lr = Learning Rate
     ax = Axis to plot the Gradient Descent
     ax1 = Axis to plot cost_history vs Iterations plot

     """
     _ = ax.plot(X,y,'b.')
     theta = np.random.randn(2,1)

     tr =0.1
     cost_history = np.zeros(n_iter)
     for i in range(n_iter):
        pred_prev = X_b.dot(theta)
        theta,h,_ = gradient_descent(X_b,y,theta,lr,1)
        pred = X_b.dot(theta)

        cost_history[i] = h[0]

        if ((i % 25 == 0) ):
            _ = ax.plot(X,pred,'r-',alpha=tr)
            if tr < 0.8:
                tr = tr+0.2
     if not ax1== None:
        _ = ax1.plot(range(n_iter),cost_history,'b.')
```

### Plot the graphs for different iterations and learning rates combination

```python
fig = plt.figure(figsize=(30,25),dpi=200)
fig.subplots_adjust(hspace=0.4, wspace=0.4)

it_lr =[(2000,0.001),(500,0.01),(200,0.05),(100,0.1)]
count =0
for n_iter, lr in it_lr:
    count += 1

    ax = fig.add_subplot(4, 2, count)
    count += 1

    ax1 = fig.add_subplot(4,2,count)

    ax.set_title("lr:{}".format(lr))
    ax1.set_title("Iterations:{}".format(n_iter))
    plot_GD(n_iter,lr,ax,ax1)

```

 See how useful it is to visualize the effect of learning rates and iterations on gradient descent. The red lines show how the gradient descent starts and then slowly gets closer to the final value

## You can always plot Indiviual graphs to zoom in

```python
_,ax = plt.subplots(figsize=(14,10))
plot_GD(100,0.1,ax)
```

# Stochastic Gradient Descent

```python
def stocashtic_gradient_descent(X,y,theta,learning_rate=0.01,iterations=10):
    '''
    X    = Matrix of X with added bias units
    y    = Vector of Y
    theta=Vector of thetas np.random.randn(j,1)
    learning_rate
    iterations = no of iterations

    Returns the final theta vector and array of cost history over no of iterations
    '''
    m = len(y)
    cost_history = np.zeros(iterations)

    for it in range(iterations):
        cost =0.0
        for i in range(m):
            rand_ind = np.random.randint(0,m)
            X_i = X[rand_ind,:].reshape(1,X.shape[1])
            y_i = y[rand_ind].reshape(1,1)
            prediction = np.dot(X_i,theta)

            theta = theta -(1/m)*learning_rate*( X_i.T.dot((prediction - y_i)))
            cost += cal_cost(theta,X_i,y_i)
        cost_history[it]  = cost

    return theta, cost_history
```

```python
lr =0.5
n_iter = 50

theta = np.random.randn(2,1)

X_b = np.c_[np.ones((len(X),1)),X]
theta,cost_history = stocashtic_gradient_descent(X_b,y,theta,lr,n_iter)

print('Theta0:          {:0.3f},\nTheta1:          {:0.3f}'.format(theta[0][0],theta[1][0]))
print('Final cost/MSE:  {:0.3f}'.format(cost_history[-1]))
```

    Theta0:          4.116,
    Theta1:          2.981
    Final cost/MSE:  51.160

```python
fig,ax = plt.subplots(figsize=(10,8))

ax.set_ylabel('{J(Theta)}',rotation=0)
ax.set_xlabel('{Iterations}')
theta = np.random.randn(2,1)

_=ax.plot(range(n_iter),cost_history,'b.')
```

# Mini Batch Gradient Descent

```python
def minibatch_gradient_descent(X,y,theta,learning_rate=0.01,iterations=10,batch_size =20):
    '''
    X    = Matrix of X without added bias units
    y    = Vector of Y
    theta=Vector of thetas np.random.randn(j,1)
    learning_rate
    iterations = no of iterations

    Returns the final theta vector and array of cost history over no of iterations
    '''
    m = len(y)
    cost_history = np.zeros(iterations)
    n_batches = int(m/batch_size)

    for it in range(iterations):
        cost =0.0
        indices = np.random.permutation(m)
        X = X[indices]
        y = y[indices]
        for i in range(0,m,batch_size):
            X_i = X[i:i+batch_size]
            y_i = y[i:i+batch_size]

            X_i = np.c_[np.ones(len(X_i)),X_i]

            prediction = np.dot(X_i,theta)

            theta = theta -(1/m)*learning_rate*( X_i.T.dot((prediction - y_i)))
            cost += cal_cost(theta,X_i,y_i)
        cost_history[it]  = cost

    return theta, cost_history
```

```python
lr =0.1
n_iter = 200

theta = np.random.randn(2,1)

theta,cost_history = minibatch_gradient_descent(X,y,theta,lr,n_iter)

print('Theta0:          {:0.3f},\nTheta1:          {:0.3f}'.format(theta[0][0],theta[1][0]))
print('Final cost/MSE:  {:0.3f}'.format(cost_history[-1]))
```

    Theta0:          4.060,
    Theta1:          3.092
    Final cost/MSE:  1171.808

```python
fig,ax = plt.subplots(figsize=(10,8))

ax.set_ylabel('{J(Theta)}',rotation=0)
ax.set_xlabel('{Iterations}')
theta = np.random.randn(2,1)

_=ax.plot(range(n_iter),cost_history,'b.')
```

```python

```


# Lab 4
## Introduction
The Euler method is a method for numerically solving a DE of the form
\begin{align}
\frac{\mathrm{d}y}{\mathrm{d}x} = f(x,y).
\end{align}

It is often necessary to solve DEs this way as analytical solutions are the exception
rather than the rule.

Euler’s method works by approximating small segments of the curve solution to the DE
with the straight-line tangent or slope of the curve. As long as we keep the segments
small enough, they will approximately match what the actual curve looks like. It requires us to ”know” an initial value $y(x_0) = y_0$ so we can start the calculation.

To calculate the first segment we start off with our known start point $(x_0, y_0)$, and calculate the end point, $(x_1, y_1)$. We can define $\Delta x$ to be some constant small distance so that we always increment the $x$ value by the same amount. Then, $\Delta y = m \Delta x$ and $(x_1, y_1)=(x_0, y_0)+(\Delta x, m\Delta x)$.

But, we also know that $m$, the slope of the line, is given by $\mathrm{d}y/\mathrm{d}x$, i.e., $f(x, y)$ evaluated at $(x_0, y_0)$. So actually, $\Delta y = f(x_0, y_0) \Delta x$.

The final step is to calculate the new point: the point at the end of the first line segment. This point is then given by $(x_1, y_1) = (x_0 + \Delta x, y_0 + f(x_0, y_0) \Delta x)$.

We then do it again to calculate $(x_2, y_2)$ using $(x_1, y_1)$ as our starting point. We then do it again to calculate $(x_3, y_3)$ using $(x_2, y_2)$ as our starting point and so on.

**Summary:** The Euler method for evaluating a DE of the form
\begin{align}
\frac{\mathrm{d}y}{\mathrm{d}x} = f(x,y).
\end{align}
involves the iterative calculation of
\begin{align}
x_{n+1} &= x_n + \Delta x\\
\text{and}\quad y_{n+1} &= y_n + f(x_n,y_n)\Delta x.
\end{align}

### Implementation

First import the necessary functions from NumPy and SciPy and set up Plotly.

```python
from numpy import arange, empty, exp
from plotly import graph_objs as go
```

Now let's write a function that implements Euler's method. We will model it on `scipy.integrate.odeint`. We will make slight changes to the parameters because we want to input $\Delta x$. Note that the string (delimited by triple quotes) immediately after the function definition is a _docstring_. It tells us what the function does and is good programming practice. The prodigious comments in the function body are not generally necessary but are included for you.

```python
def euler(func, y0, x0, xn, Dx):
    """
    Integrate an ordinary differential equation using Euler's method.

    Solves the initial value problem for systems of first order ode-s::
        dy/dx = func(y, x).

    Parameters
    ----------
    func : callable(y, x)
        Computes the derivative of y at x.
    y0 : float
        Initial condition on y.
    x0 : float
        Initial condition on x.
    xn : float
        Upper limit to value of x.
    Dx : float
        x increment.

    Returns
    -------
    x : float
        Array containing the value of x for each value of x0 + n * Dx,
        where n ranges from zero to floor( (xn - x0) / Dx ).
    y : float
        Array containing the value of y for each value of x.
    """
    x = arange(x0, xn, Dx)  # Create the x array
    y = empty(len(x))  # Create an empty y array of the same length as x
    y[0] = y0  # Set the first value of y to y0
    for n in range(len(x) - 1):  # Loop to populate the rest of the values of y
        y[n+1] = y[n] + func(y[n], x[n]) * Dx  # Euler's method

    return x, y  # Return x and y as a pair
```

First try solving
\begin{align}
\frac{\mathrm{d}y}{\mathrm{d}x} = y.
\end{align}
for $y(0)=1$ for $x$ between 1 and 5 and using $\Delta x=1$.

```python
def diff_eq(y, x):
    return y

x, y = euler(diff_eq, 1, 0, 5.01, 1)
```

Why was `xn` set to 5.01 rather than 5?

We know that the analytic solution to the above IVP is $y=\mathrm{e}^x$, so calculate that as well.

```python
x_analytic = arange(0, 5.01, 0.1)
y_analytic = exp(x_analytic)
```

Now plot them both for comparison.

```python
fig = go.Figure()
fig.add_trace(go.Scatter(x=x, y=y,
                         name='Euler'))
fig.add_trace(go.Scatter(x=x_analytic,
                         y=y_analytic,
                         name='Truth'))
fig.show('png')
```

Reproduce the comparison plot below but with $\Delta x=0.1$.

```python

```

It is possible to quantify the error in the Euler solution compared to the analytic solution. To do this you need to re-calculate the analytic solution at the same $x$ points as you calculated your Euler solution. Then you can do a Mean Squared Error (MSE) comparison between the two.

```python
x, y = euler(diff_eq, 1, 0, 5.01, 1)
y_analytic = exp(x)
((y_analytic - y)**2).mean()
```

Note that `((y_analytic - y)**2)` returned an `array` object, and then we called the `mean` method that was _bound_ to that object.

What is the MSE if $\Delta x = 0.1$?

```python

```

## Exercises

In this lab you will try Euler's method for a couple of differential equations.

1.  a. Consider the IVP
\begin{align}
\frac{\mathrm{d}y}{\mathrm{d}x} = 2x\quad\text{where}\quad y(-2)=4.
\end{align}
Calculate the Euler approximation on the interval $x=[-2,2]$ using a step size of $\Delta x = 0.5$. On the same figure, plot your approximation and the analytic solution.

```python

```

1.  b. Calculate the mean squared error (MSE) of the approximation.

```python

```

1.  c. Reproduce your plot from 1a except with $\Delta x=0.1$.

```python

```

1.  d. Recalculate the MSE.

```python

```

2.  a. The following is the DE for the arrow problem from class.
\begin{align}
\frac{\mathrm{d}y}{\mathrm{d}t} = 294\mathrm{e}^{-0.04t}-245\quad\text{where}\quad y(0)=0
\end{align}
Calculate the Euler approximation to the solution on the interval  $t=[0,10]$ with $\Delta t=0.5$. Plot your approximation and the analytic solution on the same figure.

```python

```

2.  b. Calculate the MSE of the approximation.

```python

```

2.  c. Reproduce your plot from 2a except with $\Delta t=0.1$.

```python

```

2.  d. Recalculate the MSE.

```python

```


# The $\chi^2$ Distribution

## $\chi^2$ Test Statistic

If we make $n$ ranom samples (observations) from Gaussian (Normal) distributions with known means, $\mu_i$, and known variances, $\sigma_i^2$, it is seen that the total squared deviation,

$$
\chi^2 = \sum_{i=1}^{n} \left(\frac{x_i - \mu_i}{\sigma_i}\right)^2\,,
$$

follows a $\chi^2$ distribution with $n$ degrees of freedom.

## Probability Distribution Function

The $\chi^2$ probability distribution function for $k$ degrees of freedom (the number of parameters that are allowed to vary) is given by

$$
f\left(\chi^2\,;k\right) = \frac{\displaystyle 1}{\displaystyle 2^{k/2} \,\Gamma\left(k\,/2\right)}\, \chi^{k-2}\,e^{-\chi^2/2}\,,
$$

where if there are no constrained variables the number of degrees of freedom, $k$, is equal to the number of observations, $k=n$. The p.d.f. is often abbreviated in notation from $f\left(\chi^2\,;k\right)$ to $\chi^2_k$.

A reminder that for integer values of $k$, the Gamma function is $\Gamma\left(k\right) = \left(k-1\right)!$, and that $\Gamma\left(x+1\right) = x\Gamma\left(x\right)$, and $\Gamma\left(1/2\right) = \sqrt{\pi}$.

## Mean

Letting $\chi^2=z$, and noting that the form of the Gamma function is

$$
\Gamma\left(z\right) = \int\limits_{0}^{\infty} x^{z-1}\,e^{-x}\,dx,
$$

it is seen that the mean of the $\chi^2$ distribution $f\left(\chi^2 ; k\right)$ is

$$
\begin{align}
\mu &= \textrm{E}\left[z\right] = \displaystyle\int\limits_{0}^{\infty} z\, \frac{\displaystyle 1}{\displaystyle 2^{k/2} \,\Gamma\left(k\,/2\right)}\, z^{k/2-1}\,e^{-z\,/2}\,dz \\
    &= \displaystyle \frac{\displaystyle 1}{\displaystyle \Gamma\left(k\,/2\right)} \int\limits_{0}^{\infty} \left(\frac{z}{2}\right)^{k/2}\,e^{-z\,/2}\,dz = \displaystyle \frac{\displaystyle 1}{\displaystyle \Gamma\left(k\,/2\right)} \int\limits_{0}^{\infty} x^{k/2}\,e^{-x}\,2 \,dx \\
    &= \displaystyle \frac{\displaystyle 2 \,\Gamma\left(k\,/2 + 1\right)}{\displaystyle \Gamma\left(k\,/2\right)} \\
    &= \displaystyle 2 \frac{k}{2} \frac{\displaystyle \Gamma\left(k\,/2\right)}{\displaystyle \Gamma\left(k\,/2\right)} \\
    &= k.
\end{align}
$$

## Variance

Likewise, the variance is

$$
\begin{align}
\textrm{Var}\left[z\right] &= \textrm{E}\left[\left(z-\textrm{E}\left[z\right]\right)^2\right] = \displaystyle\int\limits_{0}^{\infty} \left(z - k\right)^2\, \frac{\displaystyle 1}{\displaystyle 2^{k/2} \,\Gamma\left(k\,/2\right)}\, z^{k/2-1}\,e^{-z\,/2}\,dz \\
    &= \displaystyle\int\limits_{0}^{\infty} z^2\, f\left(z \,; k\right)\,dz - 2k\int\limits_{0}^{\infty} z\,\,f\left(z \,; k\right)\,dz + k^2\int\limits_{0}^{\infty} f\left(z \,; k\right)\,dz \\
    &= \displaystyle\int\limits_{0}^{\infty} z^2 \frac{\displaystyle 1}{\displaystyle 2^{k/2} \,\Gamma\left(k\,/2\right)}\, z^{k/2-1}\,e^{-z\,/2}\,dz - 2k^2 + k^2\\
    &= \displaystyle\int\limits_{0}^{\infty} \frac{\displaystyle 1}{\displaystyle 2^{k/2} \,\Gamma\left(k\,/2\right)}\, z^{k/2+1}\,e^{-z\,/2}\,dz - k^2\\
    &= \frac{\displaystyle 2}{\displaystyle \Gamma\left(k\,/2\right)} \displaystyle\int\limits_{0}^{\infty} \left(\frac{z}{2}\right)^{k/2+1}\,e^{-z\,/2}\,dz - k^2 = \frac{\displaystyle 2}{\displaystyle \Gamma\left(k\,/2\right)} \displaystyle\int\limits_{0}^{\infty} x^{k/2+1}\,e^{-x}\,2\,dx - k^2 \\
    &= \displaystyle \frac{\displaystyle 4 \,\Gamma\left(k\,/2 + 2\right)}{\displaystyle \Gamma\left(k\,/2\right)} - k^2 \\
    &= \displaystyle 4 \left(\frac{k}{2} + 1\right) \frac{\displaystyle \Gamma\left(k\,/2 + 1\right)}{\displaystyle \Gamma\left(k\,/2\right)} - k^2 \\
    &= \displaystyle 4 \left(\frac{k}{2} + 1\right) \frac{k}{2} - k^2 \\
    &= k^2 + 2k - k^2 \\
    &= 2k,
\end{align}
$$

such that the standard deviation is

$$
\sigma = \sqrt{2k}\,.
$$

Given this information we now plot the $\chi^2$ p.d.f. with various numbers of degrees of freedom to visualize how the distribution's behaviour

```python
import numpy as np
import scipy.stats as stats

import matplotlib.pyplot as plt
```

```python
# Plot the chi^2 distribution
x = np.linspace(0.0, 10.0, num=1000)

[plt.plot(x, stats.chi2.pdf(x, df=ndf), label=fr"$k = ${ndf}") for ndf in range(1, 7)]

plt.ylim(-0.01, 0.5)

plt.xlabel(r"$x=\chi^2$")
plt.ylabel(r"$f\left(x;k\right)$")
plt.title(r"$\chi^2$ distribution for various degrees of freedom")

plt.legend(loc="best")

plt.show();
```

## Cumulative Distribution Function

The cumulative distribution function (CDF) for the $\chi^2$ distribution is (letting $z=\chi^2$)

$$
\begin{split}
F_{\chi^2}\left(x\,; k\right) &= \int\limits_{0}^{x} f_{\chi^2}\left(z\,; k\right) \,dz \\
    &= \int\limits_{0}^{x} \frac{\displaystyle 1}{\displaystyle 2^{k/2} \,\Gamma\left(k\,/2\right)}\, z^{k/2-1}\,e^{-z/2} \,dz \\
    &= \int\limits_{0}^{x} \frac{\displaystyle 1}{\displaystyle 2 \,\Gamma\left(k\,/2\right)}\, \left(\frac{z}{2}\right)^{k/2-1}\,e^{-z/2} \,dz = \frac{1}{\displaystyle 2 \,\Gamma\left(k\,/2\right)}\int\limits_{0}^{x/2} t^{k/2-1}\,e^{-t} \,2\,dt \\
    &= \frac{1}{\displaystyle \Gamma\left(k\,/2\right)}\int\limits_{0}^{x/2} t^{k/2-1}\,e^{-t} \,dt
\end{split}
$$

Noting the form of the [lower incomplete gamma function](https://en.wikipedia.org/wiki/Incomplete_gamma_function) is

$$
\gamma\left(s,x\right) = \int\limits_{0}^{x} t^{s-1}\,e^{-t} \,dt\,,
$$

and the form of the [regularized Gamma function](https://en.wikipedia.org/wiki/Incomplete_gamma_function#Regularized_Gamma_functions_and_Poisson_random_variables) is

$$
P\left(s,x\right) = \frac{\gamma\left(s,x\right)}{\Gamma\left(s\right)}\,,
$$

it is seen that

$$
\begin{split}
F_{\chi^2}\left(x\,; k\right) &= \frac{1}{\displaystyle \Gamma\left(k\,/2\right)}\int\limits_{0}^{x/2} t^{k/2-1}\,e^{-t} \,dt \\
    &= \frac{\displaystyle \gamma\left(\frac{k}{2},\frac{x}{2}\right)}{\displaystyle \Gamma\left(\frac{k}{2}\right)} \\
    &= P\left(\frac{k}{2},\frac{x}{2}\right)\,.
\end{split}
$$

Thus, it is seen that the compliment to the CDF (the complementary cumulative distribution function (CCDF)),

$$
\bar{F}_{\chi^2}\left(x\,; k\right) = 1-F_{\chi^2}\left(x\,; k\right),
$$

represents a one-sided (one-tailed) $p$-value for observing a $\chi^2$ given a model — that is, the probability to observe a $\chi^2$ value greater than or equal to that which was observed.

```python
def chi2_ccdf(x, df):
    """The complementary cumulative distribution function

    Args:
        x: the value of chi^2
        df: the number of degrees of freedom

    Returns:
        1 - the cumulative distribution function
    """
    return 1.0 - stats.chi2.cdf(x=x, df=df)
```

```python
x = np.linspace(0.0, 10.0, num=1000)
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(14, 4.5))

for ndf in range(1, 7):
    axes[0].plot(x, stats.chi2.cdf(x, df=ndf), label=fr"$k = ${ndf}")
    axes[1].plot(x, chi2_ccdf(x, df=ndf), label=fr"$k = ${ndf}")

axes[0].set_xlabel(r"$x=\chi^2$")
axes[0].set_ylabel(r"$F\left(x;k\right)$")
axes[0].set_title(r"$\chi^2$ CDF for various degrees of freedom")

axes[0].legend(loc="best")

axes[1].set_xlabel(r"$x=\chi^2$")
axes[1].set_ylabel(r"$\bar{F}\left(x;k\right) = p$-value")
axes[1].set_title(r"$\chi^2$ CCDF ($p$-value) for various degrees of freedom")

axes[1].legend(loc="best")

plt.show();
```

## Binned $\chi^2$ per Degree of Freedom

TODO

## References

- \[1\] G. Cowan, _Statistical Data Analysis_, Oxford University Press, 1998
- \[2\] G. Cowan, "Goodness of fit and Wilk's theorem", Notes, 2013


# Working with Matrices

```python
import numpy as np
```

## Matrices and liner combinations

### Post-multiplication with vector

Matrix-vector multiplication is a linear combination of the columns of the matrix

$$
\begin{bmatrix}
1 & 2 \\
3 & 4 \\
5 & 6
\end{bmatrix}
\begin{bmatrix}
2 \\ 3
\end{bmatrix} =
2 \begin{bmatrix}
1 \\ 3 \\ 5
\end{bmatrix} +
3 \begin{bmatrix}
2 \\ 4 \\ 6
\end{bmatrix} =
\begin{bmatrix}
8 \\
18 \\
28
\end{bmatrix}
$$

$$
\begin{bmatrix}
1 & 2 \\
3 & 4 \\
5 & 6
\end{bmatrix}
\begin{bmatrix}
1 \\ 4
\end{bmatrix} =
1 \begin{bmatrix}
1 \\ 3 \\ 5
\end{bmatrix} +
4 \begin{bmatrix}
2 \\ 4 \\ 6
\end{bmatrix} =
\begin{bmatrix}
9 \\
19 \\
29
\end{bmatrix}
$$

We can stack the columns horizontally to get matrix multiplication.

$$
\begin{bmatrix}
1 & 2 \\
3 & 4 \\
5 & 6
\end{bmatrix}
\begin{bmatrix}
2 & 1 \\ 3 & 4
\end{bmatrix} =
\begin{bmatrix}
8 & 9 \\
18 & 19 \\
28 & 29
\end{bmatrix}
$$

```python
A = np.arange(1, 7).reshape((3,2))
x1 = np.array([2, 3]).reshape((2,1))
x2 = np.array([1,4]).reshape((2,1))
```

```python
A @ x1
```

    array([[ 8],
           [18],
           [28]])

```python
A @ x2
```

    array([[ 9],
           [19],
           [29]])

```python
np.c_[x1, x2]
```

    array([[2, 1],
           [3, 4]])

```python
A @ np.c_[x1, x2]
```

    array([[ 8,  9],
           [18, 19],
           [28, 29]])

### Pre-multiplication with vector

Vector-matrix multiplication is a linear combination of the rows of the matrix

$$
\begin{bmatrix}
1 & 2 & 3
\end{bmatrix}
\begin{bmatrix}
1 & 2 \\
3 & 4 \\
5 & 6
\end{bmatrix}=
1 \begin{bmatrix}
1 & 2
\end{bmatrix} +
2 \begin{bmatrix}
3 & 4
\end{bmatrix} +
3 \begin{bmatrix}
5 & 6
\end{bmatrix} =
\begin{bmatrix}
22 & 28
\end{bmatrix}
$$

$$
\begin{bmatrix}
4 & 5 & 6
\end{bmatrix}
\begin{bmatrix}
1 & 2 \\
3 & 4 \\
5 & 6
\end{bmatrix}=
4 \begin{bmatrix}
1 & 2
\end{bmatrix} +
5 \begin{bmatrix}
3 & 4
\end{bmatrix} +
6 \begin{bmatrix}
5 & 6
\end{bmatrix} =
\begin{bmatrix}
49 & 64
\end{bmatrix}
$$

We can stack the rows vertically to get matrix multiplication.

$$
\begin{bmatrix}
1 & 2 & 3 \\
4 & 5 & 4
\end{bmatrix}
\begin{bmatrix}
1 & 2 \\
3 & 4 \\
5 & 6
\end{bmatrix} =
\begin{bmatrix}
22 & 28 \\
49 & 64
\end{bmatrix}
$$

Matrix-matrix multiplication can be seen as the horizontal stacking of column operations or as the vertical stacking of row operations.

```python
y1 = np.array([1,2,3]).reshape((1,3))
y2 = np.array([4,5,6]).reshape((1,3))
```

```python
y1 @ A
```

    array([[22, 28]])

```python
y2 @ A
```

    array([[49, 64]])

```python
np.r_[y1, y2]
```

    array([[1, 2, 3],
           [4, 5, 6]])

```python
np.r_[y1, y2] @ A
```

    array([[22, 28],
           [49, 64]])

### Extract columns of a matrix by post-multiplication with standard unit column vector

```python
A
```

    array([[1, 2],
           [3, 4],
           [5, 6]])

```python
e2 = np.array([0,1]).reshape((-1,1))
```

```python
A @ e2
```

    array([[2],
           [4],
           [6]])

### Extract rows of a matrix by pre-multiplication with standard unit  row vector

```python
e2 = np.array([0,1,0]).reshape((-1, 1))
```

```python
e2.T @ A
```

    array([[3, 4]])

## Permutation matrices

From the column extraction by post-multiplication with a standard unit column vector, we generalize to permutation matrices (identity matrix with permuted columns). Post-multiplication of a matrix $A$ with a permutation matrix $P$ rearranges the columns of  $A$. To recover the original matrix, multiply with $P^T$ - i.e. $P^{-1} = P^T$ and the inverse of $P$ is its inverse, $P$ being our first example of an orthogonal matrix.

```python
A = np.arange(1, 17).reshape((4,4))
A
```

    array([[ 1,  2,  3,  4],
           [ 5,  6,  7,  8],
           [ 9, 10, 11, 12],
           [13, 14, 15, 16]])

```python
I = np.eye(4, dtype='int')
I
```

    array([[1, 0, 0, 0],
           [0, 1, 0, 0],
           [0, 0, 1, 0],
           [0, 0, 0, 1]])

```python
A @ I
```

    array([[ 1,  2,  3,  4],
           [ 5,  6,  7,  8],
           [ 9, 10, 11, 12],
           [13, 14, 15, 16]])

```python
p = I[:, [2,1,3,0]]
p
```

    array([[0, 0, 0, 1],
           [0, 1, 0, 0],
           [1, 0, 0, 0],
           [0, 0, 1, 0]])

```python
A @ p
```

    array([[ 3,  2,  4,  1],
           [ 7,  6,  8,  5],
           [11, 10, 12,  9],
           [15, 14, 16, 13]])

```python
A @ p @ p.T
```

    array([[ 1,  2,  3,  4],
           [ 5,  6,  7,  8],
           [ 9, 10, 11, 12],
           [13, 14, 15, 16]])

## Matrix partitioning

We see above that matrix multiplication can be seen as separate operations on the row or column vectors. We can actually partition matrices into blocks (not just vectors) for matrix multiplication. Suppose we want to calculate $AB$, where

\begin{align}
A = \begin{bmatrix}
1 & 0 & 1 & 0 \\
0 & 1 & 0 & 1 \\
0 & 0 & 2 & 0 \\
0 & 0 & 0 & 3
\end{bmatrix}&, & B = \begin{bmatrix}
1 & 2 & 3 & 4 \\
5 & 6 & 7 & 8 \\
0 & 0 & 1 & 0 \\
0 & 0 & 0 & 1
\end{bmatrix}
\end{align}

We can consider (say) $A$ and $B$ as each being a $2 \times 2$ matrix where each element is a $2 \times 2$ sub-matrix (or block). This simplifies the computation since many blocks are the identity or null matrix.

\begin{align}
A = \begin{bmatrix}
A_{11} & A_{12} \\
A_{21} & A_{22}
\end{bmatrix}&, & B = \begin{bmatrix}
B_{11} & B_{12} \\
B_{21} & B_{22}
\end{bmatrix}
\end{align}

and

$$
AB = \begin{bmatrix}
A_{11}B_{11} + A_{12}B_{21} & A_{11}B_{12} + A_{12}B_{22} \\
A_{21}B_{11} + A_{22}B_{22} & A_{21}B_{12} + A_{22}B_{22}
\end{bmatrix}
$$

In fact, we can see by inspection that the result will be

$$
AB = \begin{bmatrix}
B_{11} & B_{12}+I_2 \\
0_2 & A_{22}
\end{bmatrix} = \begin{bmatrix}
1 & 2 & 4 & 4 \\
5 & 6 & 7 & 9 \\
0 & 0 & 2 & 0 \\
0 & 0 & 0 & 3
\end{bmatrix}
$$

In general, any sub-block structure consistent with matrix multiplication (more formally, $A$ and $B$ are *conformable* for multiplication) is fine. In particular, the blocks do not have to be square.

```python
a11 = np.eye(2)
a12 = np.eye(2)
a21 = np.zeros((2,2))
a22 = np.diag((2,3))

b11 = np.array([
    [1,2],
    [5,6]
])
b12 = np.array([
    [3,4],
    [7,8]
])
b21 = np.zeros((2,2))
b22 = np.eye(2)
```

```python
A = np.block([
    [a11, a12],
    [a21, a22]
]).astype('int')
A
```

    array([[1, 0, 1, 0],
           [0, 1, 0, 1],
           [0, 0, 2, 0],
           [0, 0, 0, 3]])

```python
B = np.block([
    [b11, b12],
    [b21, b22]
]).astype('int')
B
```

    array([[1, 2, 3, 4],
           [5, 6, 7, 8],
           [0, 0, 1, 0],
           [0, 0, 0, 1]])

```python
A @ B
```

    array([[1, 2, 4, 4],
           [5, 6, 7, 9],
           [0, 0, 2, 0],
           [0, 0, 0, 3]])

```python
np.block([
    [a11@b11 + a12@b21, a11@b12 + a12@b22],
    [a21@b11 + a22@b21, a21@b12 + a22@b22]
]).astype('int')
```

    array([[1, 2, 4, 4],
           [5, 6, 7, 9],
           [0, 0, 2, 0],
           [0, 0, 0, 3]])


# Quadratic Equations

Consider the following equation:

\begin{equation}y = 2(x - 1)(x + 2)\end{equation}

If you multiply out the factored ***x*** expressions, this equates to:

\begin{equation}y = 2x^{2} + 2x - 4\end{equation}

Note that the highest ordered term includes a squared variable (x2).

Let's graph this equation for a range of ***x*** values:

```python
import pandas as pd

# Create a dataframe with an x column containing values to plot
df = pd.DataFrame ({'x': range(-9, 9)})

# Add a y column by applying the quadratic equation to x
df['y'] = 2*df['x']**2 + 2 *df['x'] - 4

# Plot the line
%matplotlib inline
from matplotlib import pyplot as plt

plt.plot(df.x, df.y, color="grey")
plt.xlabel('x')
plt.ylabel('y')
plt.grid()
plt.axhline()
plt.axvline()
plt.show()
```

Note that the graph shows a *parabola*, which is an arc-shaped line that reflects the x and y values calculated for the equation.

Now let's look at another equation that includes an ***x2*** term:

\begin{equation}y = -2x^{2} + 6x + 7\end{equation}

What does that look like as a graph?:

```python
import pandas as pd

# Create a dataframe with an x column containing values to plot
df = pd.DataFrame ({'x': range(-8, 12)})

# Add a y column by applying the quadratic equation to x
df['y'] = -2*df['x']**2 + 6*df['x'] + 7

# Plot the line
%matplotlib inline
from matplotlib import pyplot as plt

plt.plot(df.x, df.y, color="grey")
plt.xlabel('x')
plt.ylabel('y')
plt.grid()
plt.axhline()
plt.axvline()
plt.show()
```

Again, the graph shows a parabola, but this time instead of being open at the top, the parabola is open at the bottom.

Equations that assign a value to ***y*** based on an expression that includes a squared value for ***x*** create parabolas. If the relationship between ***y*** and ***x*** is such that ***y*** is a *positive* multiple of the ***x2*** term, the parabola will be open at the top; when ***y*** is a *negative* multiple of the ***x2*** term, then the parabola will be open at the bottom.

These kinds of equations are known as *quadratic* equations, and they have some interesting characteristics. There are several ways quadratic equations can be written, but the *standard form* for quadratic equation is:

\begin{equation}y = ax^{2} + bx + c\end{equation}

Where ***a***, ***b***, and ***c*** are numeric coefficients or constants.

Let's start by examining the parabolas generated by quadratic equations in more detail.

## Parabola Vertex and Line of Symmetry
Parabolas are symmetrical, with x and y values converging exponentially towards the highest point (in the case of a downward opening parabola) or lowest point (in the case of an upward opening parabola). The point where the parabola meets the line of symmetry is known as the *vertex*.

Run the following cell to see the line of symmetry and vertex for the two parabolas described previously (don't worry about the calculations used to find the line of symmetry and vertex - we'll explore that later):

```python
%matplotlib inline

def plot_parabola(a, b, c):
    import pandas as pd
    import numpy as np
    from matplotlib import pyplot as plt

    # get the x value for the line of symmetry
    vx = (-1*b)/(2*a)

    # get the y value when x is at the line of symmetry
    vy = a*vx**2 + b*vx + c

    # Create a dataframe with an x column containing values from x-10 to x+10
    minx = int(vx - 10)
    maxx = int(vx + 11)
    df = pd.DataFrame ({'x': range(minx, maxx)})

    # Add a y column by applying the quadratic equation to x
    df['y'] = a*df['x']**2 + b *df['x'] + c

    # get min and max y values
    miny = df.y.min()
    maxy = df.y.max()

    # Plot the line
    plt.plot(df.x, df.y, color="grey")
    plt.xlabel('x')
    plt.ylabel('y')
    plt.grid()
    plt.axhline()
    plt.axvline()

    # plot the line of symmetry
    sx = [vx, vx]
    sy = [miny, maxy]
    plt.plot(sx,sy, color='magenta')

    # Annotate the vertex
    plt.scatter(vx,vy, color="red")
    plt.annotate('vertex',(vx, vy), xytext=(vx - 1, (vy + 5)* np.sign(a)))

    plt.show()

plot_parabola(2, 2, -4)

plot_parabola(-2, 3, 5)
```

## Parabola Intercepts
Recall that linear equations create lines that intersect the **x** and **y** axis of a graph, and we call the points where these intersections occur *intercepts*. Now look at the graphs of the parabolas we've worked with so far. Note that these parabolas both have a y-intercept; a point where the line intersects the y axis of the graph (in other words, when x is 0). However, note that the parabolas have *two* x-intercepts; in other words there are two points at which the line crosses the x axis (and y is 0). Additionally, imagine a downward opening parabola with its vertex at -1, -1. This is perfectly possible, and the line would never have an x value greater than -1, so it would have *no* x-intercepts.

Regardless of whether the parabola crosses the x axis or not, other than the vertex, for every ***y*** point in the parabola, there are *two* ***x*** points; one on the right (or positive) side of the axis of symmetry, and one of the left (or negative) side. The implications of this are what make quadratic equations so interesting. When we solve the equation for ***x***, there are *two* correct answers.

Let's take a look at an example to demonstrate this. Let's return to the first of our quadratic equations, and we'll look at it in its *factored* form:

\begin{equation}y = 2(x - 1)(x + 2)\end{equation}

Now, let's solve this equation for a ***y*** value of 0. We can restate the equation like this:

\begin{equation}2(x - 1)(x + 2) = 0\end{equation}

The equation is the product of two expressions **2(x - 1)** and **(x + 2)**. In this case, we know that the product of these expressions is 0, so logically *one or both of the expressions must return 0*.

Let's try the first one:

\begin{equation}2(x - 1) = 0\end{equation}

If we distrbute this, we get:

\begin{equation}2x - 2 = 0\end{equation}

This simplifies to:

\begin{equation}2x = 2\end{equation}

Which gives us a value for *x* of **1**.

Now let's try the other expression:

\begin{equation}x + 2 = 0\end{equation}

This gives us a value for *x* of **-2**.

So, when *y* is **0**, *x* is **-2** or **1**. Let's plot these points on our parabola:

```python
import pandas as pd

# Assign the calculated x values
x1 = -2
x2 = 1

# Create a dataframe with an x column containing some values to plot
df = pd.DataFrame ({'x': range(x1-5, x2+6)})

# Add a y column by applying the quadratic equation to x
df['y'] = 2*(df['x'] - 1) * (df['x'] + 2)

# Get x at the line of symmetry (halfway between x1 and x2)
vx = (x1 + x2) / 2

# Get y when x is at the line of symmetry
vy = 2*(vx -1)*(vx + 2)

# get min and max y values
miny = df.y.min()
maxy = df.y.max()

# Plot the line
%matplotlib inline
from matplotlib import pyplot as plt

plt.plot(df.x, df.y, color="grey")
plt.xlabel('x')
plt.ylabel('y')
plt.grid()
plt.axhline()
plt.axvline()

# Plot calculated x values for y = 0
plt.scatter([x1,x2],[0,0], color="green")
plt.annotate('x1',(x1, 0))
plt.annotate('x2',(x2, 0))

# plot the line of symmetry
sx = [vx, vx]
sy = [miny, maxy]
plt.plot(sx,sy, color='magenta')

# Annotate the vertex
plt.scatter(vx,vy, color="red")
plt.annotate('vertex',(vx, vy), xytext=(vx - 1, (vy - 5)))

plt.show()
```

So from the plot, we can see that both of the values we calculated for ***x*** align with the parabola when ***y*** is 0. Additionally, because the parabola is symmetrical, we know that every pair of ***x*** values for each ***y*** value will be equidistant from the line of symmetry, so we can calculate the ***x*** value for the line of symmetry as the average of the ***x*** values for any value of ***y***. This in turn means that we know the ***x*** coordinate for the vertex (it's on the line of symmetry), and we can use the quadratic equation to calculate ***y*** for this point.

## Solving Quadratics Using the Square Root Method
The technique we just looked at makes it easy to calculate the two possible values for ***x*** when ***y*** is 0 if the equation is presented as the product two expressions. If the equation is in standard form, and it can be factored, you could do the necessary manipulation to restate it as the product of two expressions. Otherwise, you can calculate the possible values for x by applying a different method that takes advantage of the relationship between squared values and the square root.

Let's consider this equation:

\begin{equation}y = 3x^{2} - 12\end{equation}

Note that this is in the standard quadratic form, but there is no *b* term; in other words, there's no term that contains a coeffecient for  ***x*** to the first power. This type of equation can be easily solved using the square root method. Let's restate it so we're solving for ***x*** when ***y*** is 0:

\begin{equation}3x^{2} - 12 = 0\end{equation}

The first thing we need to do is to isolate the ***x2*** term, so we'll remove the constant on the left by adding 12 to both sides:

\begin{equation}3x^{2} = 12\end{equation}

Then we'll divide both sides by 3 to isolate x2:

\begin{equation}x^{2} = 4\end{equation}

No we can isolate ***x*** by taking the square root of both sides. However, there's an additional consideration because this is a quadratic equation. The ***x*** variable can have two possibe values, so we must calculate the *principle* and *negative* square roots of the expression on the right:

\begin{equation}x = \pm\sqrt{4}\end{equation}

The principle square root of 4 is 2 (because 22 is 4), and the corresponding negative root is -2 (because -22 is also 4); so *x* is **2** or **-2**.

Let's see this in Python, and use the results to calculate and plot the parabola with its line of symmetry and vertex:

```python
import pandas as pd
import math

y = 0
x1 = int(- math.sqrt(y + 12 / 3))
x2 = int(math.sqrt(y + 12 / 3))

# Create a dataframe with an x column containing some values to plot
df = pd.DataFrame ({'x': range(x1-10, x2+11)})

# Add a y column by applying the quadratic equation to x
df['y'] = 3*df['x']**2 - 12

# Get x at the line of symmetry (halfway between x1 and x2)
vx = (x1 + x2) / 2

# Get y when x is at the line of symmetry
vy = 3*vx**2 - 12

# get min and max y values
miny = df.y.min()
maxy = df.y.max()

# Plot the line
%matplotlib inline
from matplotlib import pyplot as plt

plt.plot(df.x, df.y, color="grey")
plt.xlabel('x')
plt.ylabel('y')
plt.grid()
plt.axhline()
plt.axvline()

# Plot calculated x values for y = 0
plt.scatter([x1,x2],[0,0], color="green")
plt.annotate('x1',(x1, 0))
plt.annotate('x2',(x2, 0))

# plot the line of symmetry
sx = [vx, vx]
sy = [miny, maxy]
plt.plot(sx,sy, color='magenta')

# Annotate the vertex
plt.scatter(vx,vy, color="red")
plt.annotate('vertex',(vx, vy), xytext=(vx - 1, (vy - 20)))

plt.show()
```

## Solving Quadratics Using the Completing the Square Method
In quadratic equations where there is a *b* term; that is, a term containing **x** to the first power, it is impossible to directly calculate the square root. However, with some algebraic manipulation, you can take advantage of the ability to factor a polynomial expression in the form *a2 + 2ab + b2* as a binomial *perfect square* expression in the form *(a + b)2*.

At first this might seem like some sort of mathematical sleight of hand, but follow through the steps carefull and you'll see that there's nothing up my sleeve!

The underlying basis of this approach is that a trinomial expression like this:

\begin{equation}x^{2} + 24x + 12^{2}\end{equation}

Can be factored to this:

\begin{equation}(x + 12)^{2}\end{equation}

OK, so how does this help us solve a quadratic equation? Well, let's look at an example:

\begin{equation}y = x^{2} + 6x - 7\end{equation}

Let's start as we've always done so far by restating the equation to solve ***x*** for a ***y*** value of 0:

\begin{equation}x^{2} + 6x - 7 = 0\end{equation}

Now we can move the constant term to the right by adding 7 to both sides:

\begin{equation}x^{2} + 6x = 7\end{equation}

OK, now let's look at the expression on the left: *x2 + 6x*. We can't take the square root of this, but we can turn it into a trinomial that will factor into a perfect square by adding a squared constant. The question is, what should that constant be? Well, we know that we're looking for an expression like *x2 + 2**c**x + **c**2*, so our constant **c** is half of the coefficient we currently have for ***x***. This is **6**, making our constant **3**, which when squared is **9** So we can create a trinomial expression that will easily factor to a perfect square by adding 9; giving us the expression *x2 + 6x + 9*.

However, we can't just add something to one side without also adding it to the other, so our equation becomes:

\begin{equation}x^{2} + 6x + 9 = 16\end{equation}

So, how does that help? Well, we can now factor the trinomial expression as a perfect square binomial expression:

\begin{equation}(x + 3)^{2} = 16\end{equation}

And now, we can use the square root method to find x + 3:

\begin{equation}x + 3 =\pm\sqrt{16}\end{equation}

So, x + 3 is **-4** or **4**. We isolate ***x*** by subtracting 3 from both sides, so ***x*** is **-7** or **1**:

\begin{equation}x = -7, 1\end{equation}

Let's see what the parabola for this equation looks like in Python:

```python
import pandas as pd
import math

x1 = int(- math.sqrt(16) - 3)
x2 = int(math.sqrt(16) - 3)

# Create a dataframe with an x column containing some values to plot
df = pd.DataFrame ({'x': range(x1-10, x2+11)})

# Add a y column by applying the quadratic equation to x
df['y'] = ((df['x'] + 3)**2) - 16

# Get x at the line of symmetry (halfway between x1 and x2)
vx = (x1 + x2) / 2

# Get y when x is at the line of symmetry
vy = ((vx + 3)**2) - 16

# get min and max y values
miny = df.y.min()
maxy = df.y.max()

# Plot the line
%matplotlib inline
from matplotlib import pyplot as plt

plt.plot(df.x, df.y, color="grey")
plt.xlabel('x')
plt.ylabel('y')
plt.grid()
plt.axhline()
plt.axvline()

# Plot calculated x values for y = 0
plt.scatter([x1,x2],[0,0], color="green")
plt.annotate('x1',(x1, 0))
plt.annotate('x2',(x2, 0))

# plot the line of symmetry
sx = [vx, vx]
sy = [miny, maxy]
plt.plot(sx,sy, color='magenta')

# Annotate the vertex
plt.scatter(vx,vy, color="red")
plt.annotate('vertex',(vx, vy), xytext=(vx - 1, (vy - 10)))

plt.show()
```

## Vertex Form
Let's look at another example of a quadratic equation in standard form:

\begin{equation}y = 2x^{2} - 16x + 2\end{equation}

We can start to solve this by subtracting 2 from both sides to move the constant term from the right to the left:

\begin{equation}y - 2 = 2x^{2} - 16x\end{equation}

Now we can factor out the coefficient for x2, which is **2**. 2x2 is 2 • x2, and -16x is 2 • 8x:

\begin{equation}y - 2 = 2(x^{2} - 8x)\end{equation}

Now we're ready to complete the square, so we add the square of half of the -8x coefficient on the right side to the parenthesis. Half of -8 is -4, and -42 is 16, so the right side of the equation becomes *2(x2 - 8x + 16)*. Of course, we can't add something to one side of the equation without also adding it to the other side, and we've just added 2 • 16 (which is 32) to the right, so we must also add that to the left.

\begin{equation}y - 2 + 32 = 2(x^{2} - 8x + 16)\end{equation}

Now we can simplify the left and factor out a perfect square binomial expression on the right:

\begin{equation}y + 30 = 2(x - 4)^{2}\end{equation}

We now have a squared term for ***x***, so we could use the square root method to solve the equation. However, we can also isolate ***y*** by subtracting 30 from both sides. So we end up restating the original equation as:

\begin{equation}y = 2(x - 4)^{2} - 30\end{equation}

Let's just quickly check our math with Python:

```python
from random import randint
x = randint(1,100)

2*x**2 - 16*x + 2 == 2*(x - 4)**2 - 30
```

    True

So we've managed to take the expression ***2x2 - 16x + 2*** and change it to ***2(x - 4)2 - 30***. How does that help?

Well, when a quadratic equation is stated this way, it's in *vertex form*, which is generically described as:

\begin{equation}y = a(x - h)^{2} + k\end{equation}

The neat thing about this form of the equation is that it tells us the coordinates of the vertex - it's at ***h,k***.

So in this case, we know that the vertex of our equation is 4, -30. Moreover, we know that the line of symmetry is at ***x = 4***.

We can then just use the equation to calculate two more points, and the three points will be enough for us to determine the shape of the parabola. We can simply choose any ***x*** value we like and substitute it into the equation to calculate the corresponding ***y*** value. For example, let's calculate ***y*** when x is **0**:

\begin{equation}y = 2(0 - 4)^{2} - 30\end{equation}

When we work through the equation, it gives us the answer **2**, so we know that the point 0, 2 is in our parabola.

So, we know that the line of symmetry is at ***x = h*** (which is 4), and we now know that the ***y*** value when ***x*** is 0 (***h*** - ***h***) is 2. The ***y*** value at the same distance from the line of symmetry in the negative direction will be the same as the value in the positive direction, so when ***x*** is ***h*** + ***h***, the ***y*** value will also be 2.

The following Python code encapulates all of this in a function that draws and annotates a parabola using only the ***a***, ***h***, and ***k*** values from a quadratic equation in vertex form:

```python
def plot_parabola_from_vertex_form(a, h, k):
    import pandas as pd
    import math

    # Create a dataframe with an x column a range of x values to plot
    df = pd.DataFrame ({'x': range(h-10, h+11)})

    # Add a y column by applying the quadratic equation to x
    df['y'] = (a*(df['x'] - h)**2) + k

    # get min and max y values
    miny = df.y.min()
    maxy = df.y.max()

    # calculate y when x is 0 (h+-h)
    y = a*(0 - h)**2 + k

    # Plot the line
    %matplotlib inline
    from matplotlib import pyplot as plt

    plt.plot(df.x, df.y, color="grey")
    plt.xlabel('x')
    plt.ylabel('y')
    plt.grid()
    plt.axhline()
    plt.axvline()

    # Plot calculated y values for x = 0 (h-h and h+h)
    plt.scatter([h-h, h+h],[y,y], color="green")
    plt.annotate(str(h-h) + ',' + str(y),(h-h, y))
    plt.annotate(str(h+h) + ',' + str(y),(h+h, y))

    # plot the line of symmetry (x = h)
    sx = [h, h]
    sy = [miny, maxy]
    plt.plot(sx,sy, color='magenta')

    # Annotate the vertex (h,k)
    plt.scatter(h,k, color="red")
    plt.annotate('v=' + str(h) + ',' + str(k),(h, k), xytext=(h - 1, (k - 10)))

    plt.show()

# Call the function for the example discussed above
plot_parabola_from_vertex_form(2, 4, -30)
```

It's important to note that the vertex form specifically requires a *subtraction* operation in the factored perfect square term. For example, consider the following equation in the standard form:

\begin{equation}y = 3x^{2} + 6x + 2\end{equation}

The steps to solve this are:
1. Move the constant to the left side:
\begin{equation}y - 2 = 3x^{2} + 6x\end{equation}
2. Factor the ***x*** expressions on the right:
\begin{equation}y - 2 = 3(x^{2} + 2x)\end{equation}
3. Add the square of half the x coefficient to the right, and the corresponding multiple on the left:
\begin{equation}y - 2 + 3 = 3(x^{2} + 2x + 1)\end{equation}
4. Factor out a perfect square binomial:
\begin{equation}y + 1 = 3(x + 1)^{2}\end{equation}
5. Move the constant back to the right side:
\begin{equation}y = 3(x + 1)^{2} - 1\end{equation}

To express this in vertex form, we need to convert the addition in the parenthesis to a subtraction:

\begin{equation}y = 3(x - -1)^{2} - 1\end{equation}

Now, we can use the a, h, and k values to define a parabola:

```python
plot_parabola_from_vertex_form(3, -1, -1)
```

## Shortcuts for Solving Quadratic Equations
We've spent some time in this notebook discussing how to solve quadratic equations to determine the vertex of a parabola and the ***x*** values in relation to ***y***. It's important to understand the techniques we've used, which incude:
- Factoring
- Calculating the Square Root
- Completing the Square
- Using the vertex form of the equation

The underlying algebra for all of these techniques is the same, and this consistent algebra results in some shortcuts that you can memorize to make it easier to solve quadratic equations without going through all of the steps:

### Calculating the Vertex from Standard Form
You've already seen that converting a quadratic equation to the vertex form makes it easy to identify the vertex coordinates, as they're encoded as ***h*** and ***k*** in the equation itself - like this:

\begin{equation}y = a(x - \textbf{h})^{2} + \textbf{k}\end{equation}

However, what if you have an equation in standard form?:

\begin{equation}y = ax^{2} + bx + c\end{equation}

There's a quick and easy technique you can apply to get the vertex coordinates.

1. To find ***h*** (which is the x-coordinate of the vertex), apply the following formula:
\begin{equation}h = \frac{-b}{2a}\end{equation}
2. After you've found ***h***, use it in the quadratic equation to solve for ***k***:
\begin{equation}\textbf{k} = a\textbf{h}^{2} + b\textbf{h} + c\end{equation}

For example, here's the quadratic equation in standard form that we previously converted to the vertex form:

\begin{equation}y = 2x^{2} - 16x + 2\end{equation}

To find ***h***, we perform the following calculation:

\begin{equation}h = \frac{-b}{2a}\;\;\;\;=\;\;\;\;\frac{-1 \cdot16}{2\cdot2}\;\;\;\;=\;\;\;\;\frac{16}{4}\;\;\;\;=\;\;\;\;4\end{equation}

Then we simply plug the value we've obtained for ***h*** into the quadratic equation in order to find ***k***:

\begin{equation}k = 2\cdot(4^{2}) - 16\cdot4 + 2\;\;\;\;=\;\;\;\;32 - 64 + 2\;\;\;\;=\;\;\;\;-30\end{equation}

Note that a vertex at 4,-30 is also what we previously calculated for the vertex form of the same equation:

\begin{equation}y = 2(x - 4)^{2} - 30\end{equation}

### The Quadratic Formula
Another useful formula to remember is the *quadratic formula*, which makes it easy to calculate values for ***x*** when ***y*** is **0**; or in other words:

\begin{equation}ax^{2} + bx + c = 0\end{equation}

Here's the formula:

\begin{equation}x = \frac{-b \pm \sqrt{b^{2} - 4ac}}{2a}\end{equation}

Let's apply that formula to our equation, which you may remember looks like this:

\begin{equation}y = 2x^{2} - 16x + 2\end{equation}

OK, let's plug the ***a***, ***b***, and ***c*** variables from our equation into the quadratic formula:

\begin{equation}x = \frac{--16 \pm \sqrt{-16^{2} - 4\cdot2\cdot2}}{2\cdot2}\end{equation}

This simplifes to:

\begin{equation}x = \frac{16 \pm \sqrt{256 - 16}}{4}\end{equation}

This in turn (with the help of a calculator) simplifies to:

\begin{equation}x = \frac{16 \pm 15.491933384829668}{4}\end{equation}

So our positive value for ***x*** is:

\begin{equation}x = \frac{16 + 15.491933384829668}{4}\;\;\;\;=7.872983346207417\end{equation}

And the negative value for ***x*** is:

\begin{equation}x = \frac{16 - 15.491933384829668}{4}\;\;\;\;=0.12701665379258298\end{equation}

The following Python code uses the vertex formula and the quadtratic formula to calculate the vertex and the -x and +x for y = 0, and then plots the resulting parabola:

```python
def plot_parabola_from_formula (a, b, c):
    import math

    # Get vertex
    print('CALCULATING THE VERTEX')
    print('vx = -b / 2a')

    nb = -b
    a2 = 2*a
    print('vx = ' + str(nb) + ' / ' + str(a2))

    vx = -b/(2*a)
    print('vx = ' + str(vx))

    print('\nvy = ax^2 + bx + c')
    print('vy =' + str(a) + '(' + str(vx) + '^2) + ' + str(b) + '(' + str(vx) + ') + ' + str(c))

    avx2 = a*vx**2
    bvx = b*vx
    print('vy =' + str(avx2) + ' + ' + str(bvx) + ' + ' + str(c))

    vy = avx2 + bvx + c
    print('vy = ' + str(vy))

    print ('\nv = ' + str(vx) + ',' + str(vy))

    # Get +x and -x (showing intermediate calculations)
    print('\nCALCULATING -x AND +x FOR y=0')
    print('x = -b +- sqrt(b^2 - 4ac) / 2a')

    b2 = b**2
    ac4 = 4*a*c
    print('x = ' + str(nb) + '+-sqrt(' + str(b2) + ' - ' + str(ac4) + ')/' + str(a2))

    sr = math.sqrt(b2 - ac4)
    print('x = ' + str(nb) + ' +- ' + str(sr) + ' / ' + str(a2))
    print('-x = ' + str(nb) + ' - ' + str(sr) + ' / ' + str(a2))
    print('+x = ' + str(nb) + ' + ' + str(sr) + ' / ' + str(a2))

    posx = (nb + sr) / a2
    negx = (nb - sr) / a2
    print('-x = ' + str(negx))
    print('+x = ' + str(posx))

    print('\nPLOTTING THE PARABOLA')
    import pandas as pd

    # Create a dataframe with an x column a range of x values to plot
    df = pd.DataFrame ({'x': range(round(vx)-10, round(vx)+11)})

    # Add a y column by applying the quadratic equation to x
    df['y'] = a*df['x']**2 + b*df['x'] + c

    # get min and max y values
    miny = df.y.min()
    maxy = df.y.max()

    # Plot the line
    %matplotlib inline
    from matplotlib import pyplot as plt

    plt.plot(df.x, df.y, color="grey")
    plt.xlabel('x')
    plt.ylabel('y')
    plt.grid()
    plt.axhline()
    plt.axvline()

    # Plot calculated x values for y = 0
    plt.scatter([negx, posx],[0,0], color="green")
    plt.annotate('-x=' + str(negx) + ',' + str(0),(negx, 0), xytext=(negx - 3, 5))
    plt.annotate('+x=' + str(posx) + ',' + str(0),(posx, 0), xytext=(posx - 3, -10))

    # plot the line of symmetry
    sx = [vx, vx]
    sy = [miny, maxy]
    plt.plot(sx,sy, color='magenta')

    # Annotate the vertex
    plt.scatter(vx,vy, color="red")
    plt.annotate('v=' + str(vx) + ',' + str(vy),(vx, vy), xytext=(vx - 1, vy - 10))

    plt.show()

plot_parabola_from_formula (2, -16, 2)
```

```python

```

```python

```

Basics Of Algebra by Hiren/01-07-Quadratic Equations.ipynb


---
author: Nathan Carter (ncarter@bentley.edu)
---

This answer assumes you have imported SymPy as follows.

```python
from sympy import *                   # load all math functions
init_printing( use_latex='mathjax' )  # use pretty math output
```

If your equation has just one variable, simply call `solve` on it.
Note that you may get a list of more than one solution.

```python
var( 'x' )
equation = Eq( x**2 + 3*x, -x + 9 )
solve( equation )
```

$\displaystyle \left[ -2 + \sqrt{13}, \  - \sqrt{13} - 2\right]$

Sometimes you get no solutions, which is shown as a Python empty list.

```python
solve( Eq( x+1, x+2 ) )
```

$\displaystyle \left[ \right]$

Sometimes the answers include complex numbers.

```python
solve( Eq( x**3, -1 ) )
```

$\displaystyle \left[ -1, \  \frac{1}{2} - \frac{\sqrt{3} i}{2}, \  \frac{1}{2} + \frac{\sqrt{3} i}{2}\right]$

To restrict the solution to the real numbers, use `solveset` instead,
and specify the real numbers as the domain.

```python
solveset( Eq( x**3, -1 ), domain=S.Reals )
```

$\displaystyle \left\{-1\right\}$

You can solve systems of equations by calling `solve` on them.

```python
var( 'x y' )
system = [
    Eq( x + 2*y, 1 ),
    Eq( x - 9*y, 5 )
]
solve( system )
```

$\displaystyle \left\{ x : \frac{19}{11}, \  y : - \frac{4}{11}\right\}$

database/tasks/How to solve symbolic equations/Python, using SymPy.ipynb


# Binray Heap

## Properties:
1. must be complemet binary tree: assume node at index i, then the index for parent node is arr[(i-1) / 2], left child is arr[2i + 1], right child is arr[2i + 2]
2. the key of any node is no smaller than the key of its children(max heap)

## build heap
Heapify procedure can be applied to a node only if its children nodes are heapified. So the heapification must be performed in the bottom up order.

## running time
To analyze the running time, consider what happens when we’re looking at a node x that’s
in level h (where we say that the bottom level is level 0, so the level is the height). We might have to sink x all the way to the bottom, which takes time h. But there are at most ceil(n/2^(h+1)) nodes at level h. So the total cost is only
\begin{equation}
\sum_{h=0}^{log\,n} h\cdot n/2^{h+1} \,\,\leq\,\, \sum_{h=0}^{log\,n} n\cdot h/2^{h+1} \,\,\leq\,\, O(n)
\end{equation}

```python
arr = [12, 3, 45, 66, 7, 8, 9, 11, 13]
def heapify(arr, n, i):
    largest = i
    l = 2 * i + 1
    r = 2 * i + 2
    if l < n and arr[largest] < arr[l]:
        largest = l
    if r < n and arr[largest] < arr[r]:
        largest = r
    if largest != i:
        arr[largest], arr[i] = arr[i], arr[largest]
        heapify(arr, n, largest)

def build(arr):
    n = len(arr)
    for i in range(n, -1, -1):
        heapify(arr, n, i)

build(arr)
arr
```

    [66, 13, 45, 12, 7, 8, 9, 11, 3]

## heap sort
After building a max heap, extract the elements one by one.
## running time
Time complexity for heapify is O(logn), since there are n loops, the total running time is O(nlogn)

```python
def heap_sort(arr):
    n = len(arr)
    for i in range(n - 1, -1, -1):
        arr[i], arr[0] = arr[0], arr[i]
        heapify(arr, i, 0)
heap_sort(arr)
arr
```

    [3, 7, 8, 9, 11, 12, 13, 45, 66]

## deletion, insertion, decrease key is similar to the operations shown above
for deletion, we can swap x with the last node, remove x, and then
restore heap order by sinking down. This takes O(log n) time.

for insertion, we begin by inserting x into the next open spot, then we have it swim up until we restore the heap order. This takes O(log n) time.

for decrease key, we can just decrease the key of x, and have it swim up until we
restore heap order. This takes O(log n) time.

```python

```


# Finite difference method

## Finite differences

Another method of solving boundary-value problems (and also partial differential equations, as we'll see later) involves **finite differences**, which are numerical approximations to exact derivatives.

Recall that the exact derivative of a function $f(x)$ at some point $x$ is defined as:
\begin{equation}
f^{\prime}(x) = \frac{df}{dx}(x) = \lim_{\Delta x \rightarrow 0} \frac{f(x+\Delta x) - f(x)}{\Delta x}
\end{equation}

So, we can *approximate* this derivative using a finite difference (rather than an infinitesimal difference as in the exact derivative):
\begin{equation}
f^{\prime}(x) \approx \frac{f(x+\Delta x) - f(x)}{\Delta x}
\end{equation}
which involves some error. This is a **forward difference** for approximating the first derivative.
We can also approximate the first derivative using a **backward difference**:
\begin{equation}
f^{\prime}(x) \approx \frac{f(x) - f(x - \Delta x)}{\Delta x}
\end{equation}

To understand the error involved in these differences, we can use Taylor's theorem to obtain Taylor series expansions:
\begin{align}
f(x + \Delta x) &= f(x) + \Delta x \, f^{\prime}(x) + \Delta x^2 \frac{1}{2!} f^{\prime\prime}(x) + \cdots \\
\rightarrow \frac{f(x + \Delta x) - f(x)}{\Delta x} &= f^{\prime}(x) + \mathcal{O}\left( \Delta x \right) \\
f(x - \Delta x) &= f(x) - \Delta x \, f^{\prime}(x) + \Delta x^2 \frac{1}{2!} f^{\prime\prime}(x) + \cdots \\
\rightarrow \frac{f(x) - f(x - \Delta x)}{\Delta x} &= f^{\prime}(x) + \mathcal{O}\left( \Delta x \right) \\
\end{align}
where the $\mathcal{O}()$ notation stands for "order of magnitude of". So, we can see that each of these approximations is *first-order accurate*.

## Second-order finite differences

We can obtain higher-order approximations for the first derivative, and an approximations for the second derivative, by combining these Taylor series expansions:
\begin{align}
f(x + \Delta x) &= f(x) + \Delta x \, f^{\prime}(x) + \Delta x^2 \frac{1}{2!} f^{\prime\prime}(x) + \mathcal{O}\left( \Delta x^3 \right) \\
f(x - \Delta x) &= f(x) - \Delta x \, f^{\prime}(x) + \Delta x^2 \frac{1}{2!} f^{\prime\prime}(x) + \mathcal{O}\left( \Delta x^3 \right)
\end{align}

Subtracting the Taylor series for $f(x+\Delta x)$ by that for $f(x-\Delta x)$ gives:
\begin{align}
f(x + \Delta x) - f(x - \Delta x) &= 2 \Delta x \, f^{\prime}(x) + \mathcal{O}\left( \Delta x^3 \right) \\
f^{\prime}(x) &= \frac{f(x + \Delta x) - f(x - \Delta x)}{2 \Delta x} + \mathcal{O}\left( \Delta x^2 \right)
\end{align}
which is a *second-order accurate* approximation for the first derivative.

Adding the Taylor series for $f(x+\Delta x)$ to that for $f(x-\Delta x)$ gives:
\begin{align}
f(x + \Delta x) + f(x - \Delta x) &= 2 f(x) + \Delta x^2 f^{\prime\prime}(x) + \mathcal{O}\left( \Delta x^3 \right) \\
f^{\prime\prime}(x) &= \frac{f(x + \Delta x) - 2 f(x) + f(x - \Delta x)}{\Delta x^2} + \mathcal{O}\left( \Delta x^2 \right)
\end{align}
which is a *second-order accurate* approximation for the second derivative.

## Solving ODEs with finite differences

We can use finite differences to solve ODEs by substituting them for exact derivatives, and then applying the equation at discrete locations in the domain. This gives us a system of simultaneous equations to solve.

For example, let's consider the ODE
\begin{equation}
y^{\prime\prime} + x y^{\prime} - x y = 2 x \;,
\end{equation}
with the boundary conditions $y(0) = 1$ and $y(2) = 8$.

First, we *discretize* the continuous domain: divide it into a number of discrete segments. For now, let's choose $\Delta x = 0.5$, which creates four segments and thus five points: $x_1 = 0, x_2 = 0.5, x_3 = 1.0, x_4 = 1.5, x_5 = 2.0$.

Our goal is then to find approximate values of $y(x)$ at these points: $y_1$ through $y_5$. So, we have five unknowns, and need five equations to solve for them. We can use the ODE to provide these equations, by replacing the derivatives with finite differences, and applying the equation at particular discrete locations.

Recall that $y(x)$ is a function just like $f(x)$, and so we can apply the above finite difference equations to $y(x)$ and $y(x+\Delta x)$. Now that we have points, or nodes, at locations separated by $\Delta x$, we can consider a point $x_i$ where $y(x_i) = y_i$, $y(x_i + \Delta x) = y(x_{i+1}) = y_{i+1}$, and $y(x_i - \Delta x) = y(x_{i-1}) = y_{i-1}$.

To do this, we'll follow a few steps:

1.) Replace exact derivatives in the original ODE with finite differences, and apply the equation at a particular location $(x_i, y_i)$.

For our example, this gives:
\begin{equation}
\frac{y_{i+1} - 2y_i + y_{i-1}}{\Delta x^2} + x_i \left( \frac{y_{i+1} - y_{i-1}}{2 \Delta x}\right) - x_i y_i = 2 x_i
\end{equation}
which applies at location $(x_i, y_i)$.

2.) Next, rearrange the equation into a *recursion formula*:
\begin{equation}
y_{i-1} \left(1 - x_i \frac{\Delta x}{2}\right) + y_i \left( -2 -\Delta x^2 x_i \right) + y_{i+1} \left(1 + x_i \frac{\Delta x}{2}\right) = 2 x_i \Delta x^2
\end{equation}
We can use this equation to get an equation for each of the interior points in the domain.

For the first and last points—the boundary points—we already have equations, given by the boundary conditions.

3.) Set up system of linear equations

Applying the recursion formula to the interior points, and the boundary conditions for the boundary points, we can get a system of simultaneous linear equations:
\begin{align}
y_1 &= 1 \\
y_1 (0.875) + y_2 (-2.125) + y_3 (1.125) &= 0.25 \\
y_2 (0.75) + y_3 (-2.25) + y_4 (1.25) &= 0.5 \\
y_3 (0.625) + y_4 (-2.375) + y_5 (1.375) &= 0.75 \\
y_5 &= 8
\end{align}

This is a system of five equations and five unknowns, which we can solve! But, solving using substitution would be painful, so let's represent this system of equations using a matrix and vectors:
\begin{equation}
\begin{bmatrix}
1 & 0 & 0 & 0 & 0 \\
0.875 & -2.125 & 1.125 & 0 & 0 \\
0 & 0.75 & -2.25 & 1.25 & 0 \\
0 & 0 & 0.625 & -2.375 & 1.375 \\
0 & 0 & 0 & 0 & 1
\end{bmatrix}
\begin{bmatrix} y_1 \\ y_2 \\ y_3 \\ y_4 \\ y_5 \end{bmatrix} =
\begin{bmatrix} 1 \\ 0.25 \\ 0.5 \\ 0.75 \\ 8 \end{bmatrix}
\end{equation}
or, more compactly, $A \mathbf{y} = \mathbf{b}$.

4.) Solve the linear system of equations

The final step is just to solve. We can do this in Matlab with `y = A \ b`. (This is equivalent to `y = inv(A)*b`, but faster.)

```matlab
A = [1.0 0 0 0 0;
     0.875 -2.125 1.125 0 0;
         0 0.75 -2.25 1.25 0;
         0 0 0.625 -2.375 1.375;
         0 0 0 0 1];

b = [1.0; 0.25; 0.5; 0.75; 8.0];

x = [0 : 0.5 : 2];
y = A \ b;
plot(x, y, 'o-');
```

### Matlab implementation

Of course, all of this will be easier if we implement in Matlab in a general way. We'll use a `for` loop to populate the coefficient matrix $A$ and right-hand-side vector $\mathbf{b}$:

```matlab
clear all; clc

dx = 0.5;
x = [0 : dx : 2];
n = length(x);
A = zeros(n,n); b = zeros(n,1);

for i = 1 : n
    if i == 1
        A(1,1) = 1;
        b(1) = 1;
    elseif i == n
        A(n,n) = 1;
        b(n) = 8;
    else
        A(i, i-1) = 1 - x(i)*dx/2;
        A(i, i) = -2 - x(i)*dx^2;
        A(i, i+1) = 1 + x(i)*dx/2;
        b(i) = 2*x(i)*dx^2;
    end
end
y = A \ b;
plot(x, y, 'o-')
```

This looks good, but we can get a more-accurate solution by reducing our step size $\Delta x$:

```matlab
clear all; clc

dx = 0.001;
x = [0 : dx : 2];
n = length(x);
A = zeros(n,n); b = zeros(n,1);

for i = 1 : n
    if i == 1
        A(1,1) = 1;
        b(1) = 1;
    elseif i == n
        A(n,n) = 1;
        b(n) = 8;
    else
        A(i, i-1) = 1 - x(i)*dx/2;
        A(i, i) = -2 - x(i)*dx^2;
        A(i, i+1) = 1 + x(i)*dx/2;
        b(i) = 2*x(i)*dx^2;
    end
end
y = A \ b;
plot(x, y)
```

## Boundary conditions

We will encounter four main kinds of boundary conditions. Consider the ODE $y^{\prime\prime} + y = 0$, on the domain $0 \leq x \leq L$.

- First type, or Dirichlet, boundary conditions specify fixed values of $y$ at the boundaries: $y(0) = a$ and $y(L) = b$.
- Second type, or Neumann, boundary conditions specify values of the derivative at the boundaries: $y^{\prime}(0) = a$ and $y^{\prime}(L) = b$.
- Third type, or Robin, boundary conditions specify a linear combination of the function value and its derivative at the boundaries: $a \, y(0) + b \, y^{\prime}(0) = g(0)$ and $a \, y(L) + b \, y^{\prime}(L) = g(L)$, where $g(x)$ is some function.
- Mixed boundary conditions, which combine any of these three at the different boundaries. For example, we could have $y(0) = a$ and $y^{\prime}(L) = b$.

Whichever type of boundary condition we are dealing with, the goal will be to construct an equation representing the boundary condition to incorporate in our system of equations.

If we have a fixed value boundary condition, such as $y(0) = a$, then this equation is straightforward:
\begin{equation}
y_1 = a
\end{equation}
where $y_1$ is the first point in the grid of points, corresponding to $x_1 = 0$. (We saw this in the example above.) In Matlab, we can implement this equation with
```OCTAVE
A(1,1) = 1;
b(1) = a;
```

If we have a fixed derivative boundary condition, such as $y^{\prime}(0) = 0$, then we need to use a finite difference to represent the derivative. When the boundary condition is at the starting location, $x=0$, the easiest way to do this is with a **forward difference**:
\begin{align}
y^{\prime}(0) \approx \frac{y_2 - y_1}{\Delta x} &= 0 \\
-y_1 + y_2 &= 0
\end{align}
We can implement this in Matlab with
```OCTAVE
A(1,1) = -1;
A(1,2) = 1;
b(1) = 0;
```

When we have this sort of derivative boundary condition at the right side of the domain, at $x=L$, then we can use a **backward difference** to represent the derivative:
\begin{align}
y^{\prime}(L) \approx \frac{y_n - y_{n_1}}{\Delta x} &= 0 \\
-y_{n-1} + y_n &= 0
\end{align}
where $y_n$ is the final point ($x_n = L$) and $y_{n-1}$ is the second-to-last point ($x_{n-1} = L - \Delta x$). We can implement this in Matlab with
```OCTAVE
A(n,n-1) = -1;
A(n,n) = 1;
b(n) = 0;
```

If we have a linear combination of a fixed value and fixed derivative, like $a \, y(0) + b \, y^{\prime}(0) = c$, then we can combine the above approaches using a forward difference:
\begin{align}
a y(0) + b y^{\prime}(0) \approx a y_1 + b \frac{y_2 - y_1}{\Delta x} &= c \\
(a \Delta x - b) y_1 + b y_2 &= c \Delta x
\end{align}
and in Matlab:
```OCTAVE
A(1,1) = a*dx - b;
A(1,2) = b;
b(1) = c * dx;
```

### Using central differences for derivative BCs

When a boundary condition involves a derivative, we can use a *central difference* to approximate the first derivative; this is more accurate than a forward or backward difference.

Consider the formula for a central difference at $x=0$, applied for the boundary condition $y^{\prime}(0) = 0$:
\begin{align}
y^{\prime}(0) \approx \frac{y_2 - y_0}{2 \Delta x} &= 0 \\
y_0 &= y_2
\end{align}
where $y_0$ is an imaginary, or ghost, node *outside* the domain. We can't actually keep this point in our implementation, because it isn't a real point.

We still need an equation *for* the point at the boundary, $y_1$. To get this, we'll apply the regular recursion formula, normally used at interior points:
\begin{align}
a y_{i-1} + b y_i + c y_{i+1} = f(x_i) \\
a y_0 + b y_1 + c y_2 = f(x_1) \;,
\end{align}
where $a$, $b$, $c$, and $f(x)$ depend on the problem. Normally we wouldn't use this at the boundary node, $y_1$, because it references a point outside the domain to the left—but we have an equation for that! From above, based on the boundary condition, we have $y_0 = y_2$. If we incorporate that into the recursion formula, we can eliminate the ghost node $y_0$:
\begin{align}
a y_2 + b y_1 + c y_2 &= f(x_1) \\
b y_1 + (a + c) y_2 &= f(x_1) \,
\end{align}
which is the equation we can actually use at the boundary point.
In Matlab, this looks like
```OCTAVE
A(1,1) = b;
A(1,2) = a + c;
b(1) = f(x(1));
```

## Example: nonlinear BVP

So far we've seen how to handle a linear boundary value problem, but what if we have a **nonlinear** BVP? This is going to be trickier, because our work so far relies on using linear algebra to solve the system of (linear) equations.

For example, consider the 2nd-order ODE
\begin{equation}
y^{\prime\prime} = 3y + x^2 + 100 y^2
\end{equation}
with the boundary conditions $y(0) = y(1) = 0$. This is nonlinear, due to the $y^3$ term on the right-hand side.

To solve this, let's first convert it into a discrete form, by replacing the second derivative with a finite difference and any $x$/$y$ present with $x_i$ and $y_i$. We'll also move any constants (i.e., terms that don't contain $y_i$) and the nonlinear term to the right-hand side:
\begin{equation}
\frac{y_{i-1} - 2y_i + y_{i+1}}{\Delta x^2} - 3y_i = x_i^2 + 100 y_i^2
\end{equation}
where the boundary conditions are now $y_1 = 0$ and $y_n = 0$, with $n$ as the number of grid points. We can rearrange and simplify into our recursion formula:
\begin{equation}
y_{i-1} + y_i \left( -2 - 3 \Delta x^2 \right) + y_{i+1} = x_i^2 \Delta x^2 + 100 \Delta x^2 y_i^2
\end{equation}

The question is: how do we solve this now? The nonlinear term involving $y_i^3$ on the right-hand side complicates things, but we know how to set up and solve this *without* the nonlinear term. We can use an approach known as **successive iteration**:

1. Solve the ODE without the nonlinear term to get an initial "guess" to the solution for $y$.
2. Then, incorporate that guess solution in the nonlinear term on the right-hand side, treating it as a constant. We can call this $y_{\text{old}}$. Then, solve the full system for a new $y$ solution.
3. Check whether the new $y$ matches $y_{\text{old}}$ with some tolerance. For example, check whether $\max\left(\left| y - y_{\text{old}} \right| \right) < $ some tolerance, such as $10^{-6}$. If this is true, then we can consider the solution *converged*. If it is not true, then set $y_{\text{old}} = y$, and repeat the process starting at step 2.

Let's implement that process in Matlab:

```matlab
%% Initial setup
clear all; clc

dx = 0.01;
x = 0 : dx : 1;
n = length(x);

A = zeros(n, n);
b = zeros(n, 1);

%% First, solve the problem without the nonlinear term:
for i = 1 : n
    if i == 1  % x = 0 boundary condition
        A(1,1) = 1;
        b(1) = 0;
    elseif i == n % x = L boundary condition
        A(n,n) = 1;
        b(n) = 0;
    else  % interior nodes, use recursion formula
        A(i, i-1) = 1;
        A(i, i) = -2 - 3*dx^2;
        A(i, i+1) = 1;
        b(i) = x(i)^2 * dx^2;
    end
end
% get solution without nonlinear term
y = A \ b;

plot(x, y, '--'); hold on

%% Now, set up iterative process to solve while incorporating nonlinear terms
iter = 1;
y_old = zeros(n, 1);
while max(abs(y - y_old)) > 1e-6
    y_old = y;
    % A matrix is not changed, but the b vector does
    for i = 2 : n - 1
        b(i) = x(i)^2 * dx^2 + 100*(dx^2)*(y_old(i)^2);
    end

    y = A \ b;
    iter = iter + 1;
end

fprintf('Number of iterations: %d\n', iter);
plot(x, y)
legend('Initial solution', 'Final solution')
```

Another option is just to set our "guess" for the $y$ solution to be zero, rather than solve the problem in two steps:

```matlab
%% Initial setup
clear all; clc

dx = 0.01;
x = 0 : dx : 1;
n = length(x);

A = zeros(n, n);
b = zeros(n, 1);

%% Set up the coefficient matrix, which does not change
for i = 1 : n
    if i == 1  % x = 0 boundary condition
        A(1,1) = 1;
        b(1) = 0;
    elseif i == n % x = L boundary condition
        A(n,n) = 1;
        b(n) = 0;
    else  % interior nodes, use recursion formula
        A(i, i-1) = 1;
        A(i, i) = -2 - 3*dx^2;
        A(i, i+1) = 1;
        b(i) = x(i)^2 * dx^2;
    end
end

% just use zeros as our initial guess for the solution
y = zeros(n, 1);

%% Successive iteration
iter = 1;
y_old = 100 * rand(n, 1); % setting this to some random values, just to enter the while loop
while max(abs(y - y_old)) > 1e-6
    y_old = y;
    % A matrix is not changed, but the b vector does
    for i = 2 : n - 1
        b(i) = x(i)^2 * dx^2 + 100*(dx^2)*(y_old(i)^2);
    end

    y = A \ b;
    iter = iter + 1;
end

fprintf('Number of iterations: %d\n', iter);
```

    Number of iterations: 17

This made our process take slightly more iterations, because the initial guess was slightly further away from the final solution. For other problems, having a bad initial guess could make the process take much longer, so coming up with a good initial guess may be important.

## Example: heat transfer through a fin

Let's now consider a more complicated example: heat transfer through an extended surface (a fin).

:::{figure-md} fig-fin

Geometry of a heat transfer fin
:::

In this situation, we have the temperature of the body $T_b$, the temperature of the ambient fluid $T_{\infty}$; the length $L$, width $w$, and thickness $t$ of the fin; the thermal conductivity of the fin material $k$; and convection heat transfer coefficient $h$.

The boundary conditions can be defined in different ways, but generally we can say that the temperature of the fin at the wall is the same as the body temperature, and that the fin is insulated at the tip. This gives us
\begin{align}
T(x=0) &= T_b \\
q(x=L) = 0 \rightarrow \frac{dT}{dx} (x=0) &= 0
\end{align}

Our goal is to solve for the temperature distribution $T(x)$. To do this, we need to set up a governing differential equation. Let's do a control volume analysis of heat transfer through the fin:

:::{figure-md} fig-control-volume

Control volume for heat transfer through the fin
:::

Given a particular volumetric slice of the fin, we can define the heat transfer rates of conduction through the fin and convection from the fin to the air:
\begin{align}
q_{\text{conv}} &= h P \left( T - T_{\infty} \right) dx \\
q_{\text{cond}, x} &= -k A_c \left(\frac{dT}{dx}\right)_{x} \\
q_{\text{cond}, x+\Delta x} &= -k A_c \left(\frac{dT}{dx}\right)_{x+\Delta x} \;,
\end{align}
where $P$ is the perimeter (so that $P \, dx$ is the heat transfer area to the fluid) and $A_c$ is the cross-sectional area.

Performing a balance through the control volume:
\begin{align}
q_{\text{cond}, x+\Delta x} &= q_{\text{cond}, x} - q_{\text{conv}} \\
-k A_c \left(\frac{dT}{dx}\right)_{x+\Delta x} &= -k A_c \left(\frac{dT}{dx}\right)_{x} - h P \left( T - T_{\infty} \right) dx \\
-k A_c \frac{\left.\frac{dT}{dx}\right|_{x+\Delta x} - \left.\frac{dT}{dx}\right|_{x}}{dx} &= -h P ( T - T_{\infty} ) \\
\lim_{\Delta x \rightarrow 0} : -k A_c \left. \frac{d^2 T}{dx^2} \right|_x &= -h P (T - T_{\infty}) \\
\frac{d^2 T}{dx^2} &= \frac{h P}{k A_c} (T - T_{\infty}) \\
\frac{d^2 T}{dx^2} &= m^2 (T - T_{\infty})
\end{align}
then we have as a governing equation
\begin{equation}
\frac{d^2 T}{dx^2} - m^2 (T - T_{\infty}) = 0 \;,
\end{equation}
where $m^2 = (h P)/(k A_c)$.

We can obtain an exact solution for this ODE. For convenience, let's define a new variable, $\theta$, which is a normalized temperature:
\begin{equation}
\theta \equiv T - T_{\infty}
\end{equation}
where $\theta^{\prime} = T^{\prime}$ and $\theta^{\prime\prime} = T^{\prime\prime}$.
This gives us a new governing equation:
\begin{equation}
\theta^{\prime\prime} - m^2 \theta = 0 \;.
\end{equation}
This is a 2nd-order homogeneous ODE, which looks a lot like $y^{\prime\prime} + a y = 0$. The exact solution is then
\begin{align}
\theta(x) &= c_1 e^{-m x} + c_2 e^{m x} \\
T(x) &= T_{\infty} + c_1 e^{-m x} + c_2 e^{m x}
\end{align}
We'll use this to look at the accuracy of a numerical solution, but we will not be able to find an exact solution for more complicated versions of this problem.

We can also solve this numerically using the finite difference method. Let's replace the derivative with a finite difference:
\begin{align}
\frac{d^2 T}{dx^2} - m^2 (T - T_{\infty}) &= 0 \\
\frac{T_{i-1} - 2T_i + T_{i+1}}{\Delta x^2} - m^2 \left( T_i - T_{\infty} \right) &= 0
\end{align}
which we can rearrange into a recursion formula:
\begin{equation}
T_{i-1} + T_i \left( -2 - \Delta x^2 m^2 \right) + T_{i+1} = -m^2 \Delta x^2 \, T_{\infty}
\end{equation}
This gives us an equation for all the interior nodes; we can use the above boundary conditions to get equations for the boundary nodes. For the boundary condition at $x=L$, $T^{\prime}(x=L) = 0$, let's use a *backward difference*:
\begin{align}
T_1 &= T_b \\
\frac{T_n - T_{n-1}}{\Delta x} = 0 \rightarrow - T_{n-1} + T_n &= 0
\end{align}

Combining all these equations, we can construct a linear system: $A \mathbf{T} = \mathbf{b}$.

### Heat transfer with radiation

Let's now consider a more-complicated case, where we also have radiation heat transfer occuring along the length of the fin. Now, our governing ODE is
\begin{equation}
\frac{d^2 T}{dx^2} - \frac{h P}{k A_c} \left(T - T_{\infty}\right) - \frac{\sigma \epsilon P}{h A_c} \left(T^4 - T_{\infty}^4 \right) = 0
\end{equation}

This is a bit trickier to solve because of the nonlinear term involving $T^4$. But, we can handle it via the iterative solution method discussed above.


# Final project
The Allen–Cahn equation (after John W. Cahn and Sam Allen) is a reaction–diffusion equation of mathematical physics which describes the process of phase separation in multi-component alloy systems, including order-disorder transitions.

The equation describes the time evolution of a scalar-valued state variable $\eta$  on a domain $\Omega=[0,1]$  during a time interval $[0,T]$, and is given (in one dimension) by:

$$
\frac{\partial \eta}{\partial t} - \varepsilon^2 \eta'' + f'(\eta) = 0, \qquad \eta'(0, t) = \eta'(1, t) = 0,\qquad\eta(x,0) = \eta_0(x)
$$

where $f$ is a double-well potential, $\eta_0$ is the initial condition, and $\varepsilon$ is the characteristic width of the phase transition.

This equation is the L2 gradient flow of the Ginzburg–Landau free energy functional, and it is closely related to the Cahn–Hilliard equation.

A typical example of double well potential is given by the following function

$$
f(\eta) = \eta^2(\eta-1)^2
$$

which has two minima in $0$ and $1$ (the two wells, where its value is zero), one local maximum in $0.5$, and it is always greater or equal than zero.

The two minima above behave like "attractors" for the phase $\eta$. Think of a solid-liquid phase transition (say water+ice) occupying the region $[0,1]$. When $\eta = 0$, then the material is liquid, while when $\eta = 1$ the material is solid (or viceversa).

Any other value for $\eta$ is *unstable*, and the equation will pull that region towards either $0$ or $1$.

Discretisation of this problem can be done by finite difference in time. For example, a fully explicity discretisation in time would lead to the following algorithm.

We split the interval $[0,T]$ in `n_steps` intervals, of dimension `dt = T/n_steps`. Given the solution at time `t[k] = k*dt`, it i possible to compute the next solution at time `t[k+1]` as

$$
\eta_{k+1} = \eta_{k} + \Delta t \varepsilon^2 \eta_k'' - \Delta t f'(\eta_k)
$$

Such a solution will not be stable. A possible remedy that improves the stability of the problem, is to treat the linear term $\Delta t \varepsilon^2 \eta_k''$ implicitly, and keep the term $-f'(\eta_k)$ explicit, that is:

$$
\eta_{k+1} - \Delta t \varepsilon^2 \eta_k'' = \eta_{k} - \Delta t f'(\eta_k)
$$

Grouping together the terms on the right hand side, this problem is identical to the one we solved in the python notebook number 9, with the exception of the constant $\Delta t \varepsilon^2$ in front the stiffness matrix.

In particular, given a set of basis functions $v_i$, representing $\eta = \eta^j v_j$ (sum is implied), we can solve the problem using finite elements by computing

$$
\big((v_i, v_j) + \Delta t \varepsilon^2  (v_i', v_j')\big) \eta^j_{k+1} = \big((v_i, v_j) \eta^j_{k} - \Delta t (v_i, f'(\eta_k)\big)
$$
where a sum is implied over $j$ on both the left hand side and the right hand side. Let us remark that while writing this last version of the equation we moved from a forward Euler scheme to a backward Euler scheme for the second spatial derivative term: that is, we used $\eta^j_{k+1}$ instead of $\eta^j_{k}$.

This results in a linear system

$$
A x = b
$$

where

$$
A_{ij} = M_{ij}+ \Delta t \varepsilon^2 K_{ij} = \big((v_i, v_j) + \Delta t \varepsilon^2  (v_i', v_j')\big)
$$

and

$$
b_i = M_{ij} \big(\eta_k^j - \Delta t f'(\eta_k^j)\big)
$$

where we simplified the integration on the right hand side, by computing the integral of the interpolation of $f'(\eta)$.

## Step 1

Write a finite element solver, to solve one step of the problem above, given the solution at the previous time step, using the same techniques used in notebook number 9.

In particular:

1. Write a function that takes in input a vector representing $\eta$, an returns a vector containing $f'(\eta)$. Call this function `F`.

2. Write a function that takes in input a vector of support points of dimension `ndofs` and the degree `degree` of the polynomial basis, and returns a list of basis functions (piecewise polynomial objects of type `PPoly`) of dimension `ndofs`, representing the interpolatory spline basis of degree `degree`

3. Write a function that, given a piecewise polynomial object of type `PPoly` and a number `n_gauss_quadrature_points`, computes the vector of global_quadrature_points and global_quadrature_weights, that contains replicas of a Gauss quadrature formula with `n_gauss_quadrature_points` on each of the intervals defined by `unique(PPoly.x)`

4. Write a function that, given the basis and the quadrature points and weights, returns the two matrices $M$ and $K$

## Step 2

Solve the Allen-Cahan equation on the interval $[0,1]$, from time $t=0$ and time $t=1$, given a time step `dt`, a number of degrees of freedom `ndofs`, and a polynomial degree `k`.

1. Write a function that takes the initial value of $\eta_0$ as a function, eps, dt, ndofs, and degree, and returns a matrix of dimension `(int(T/dt), ndofs)`  containing all the coefficients $\eta_k^i$ representing the solution, and the set of basis functions used to compute the solution

2. Write a function that takes all the solutions `eta`, the basis functions, a stride number `s`, and a resolution `res`, and plots on a single plot the solutions $\eta_0$, $\eta_s$, $\eta_{2s}$, computed on `res` equispaced points between zero and one

## Step 3

Solve the problem for all combinations of

1. eps = [01, .001]

2. ndofs = [16, 32, 64, 128]

3. degree = [1, 2, 3]

3. dt = [.25, .125, .0625, .03125, .015625]

with $\eta_0 = \sin(2 \pi x)+1$.

Plot the final solution at $t=1$ in all cases. What do you observe? What happens when you increase ndofs and keep dt constant?

## Step 4 (Optional)

Instead of solving the problem explicitly, solve it implicitly, by using backward euler method also for the non linear term. This requires the solution of a Nonlinear problem at every step. Use scipy and numpy methods to solve the non linear iteration.

```python
%pylab inline
import sympy as sym
import scipy
from scipy.interpolate import *
from scipy.integrate import *
```

```python
# Step 1.1

def F(eta):
    # Fill in with the derivative of the double well potential function
    return eta
```

```python
# Step 1.2

def compute_basis_functions(support_points, degree):
    # Insert here what was in notebook 9

    # after you have computed the basis, return it
    # return basis
    return
```

```python
# Step 1.3

def compute_global_quadrature(basis, n_gauss_quadrature_points):
    # Create a Gauss quadrature formula with n_gauss_quadrature_points, extract the intervals from basis (i.e., unique(basis.x)), and
    # create len(x)-1 shifted and scaled Gauss quadrature formulas that can be used to integrate on each interval. Put all of these
    # together, and return the result

    # return gloabl_quadrature, global_weights
    return
```

```python
# Step 1.4

def compute_system_matrices(basis, gloabl_quadrature, global_weights):
    # Compute the matrices M_ij = (v_i, v_j) and K_ij = (v_i', v_j') and return them

    # return M, K
    return
```

```python
# Step 2.1

def solve_allen_cahan(eta_0_function, eps, dt, ndofs, degree):
    # put together all the above functions, loop over time, and produce the result matrix eta, containing the solution at all points

    #return eta, basis
```

```python
# Step 2.2

def plot_solution(eta, basis, stride, resolution):
    # plot eta[::stride], on x = linspace(0,1,resolution)
    x = linspace(0,1,resolution)

    # plot(...)
```


## Exercise 2.5
In this exercise, we prove that the beta distribution, given by

$$
\mathrm{Beta}(\mu|a, b) = \frac{\Gamma(a+b)}{\Gamma(a)\Gamma(b)}\mu^{a-1}(1-\mu)^{b-1},
$$

is correctly normalized, so that $\int_0^1\mathrm{Beta}(\mu|a, b)d\mu = 1$.

### Solution
This is equivalent to showing that

$$
\int_0^1\mu^{a-1}(1-\mu)^{b-1}d\mu = \frac{\Gamma(a+b)}{\Gamma(a)\Gamma(b)}
$$

From the definition of the gamma function, we have

$$
\Gamma(a)\Gamma(b) = \int_0^\infty\exp(-x)x^{a-1}dx\int_0^\infty\exp(-y)y^{b-1}dy
$$

Let $t = y + x$, we obtain

\begin{align}
\Gamma(a)\Gamma(b) & = \int_0^\infty x^{a-1}\left\{\int_x^\infty \exp(-t)(t-x)^{b-1}\mathrm{d}t\right\}\mathrm{d}x \\
& = \int_0^\infty\int_0^t x^{a-1}\exp(-t)(t-x)^{b-1}\mathrm{d}x\mathrm{d}t
\end{align}

Change the variables in the $x$ integral using $x=t\mu$ to give

\begin{aligned}
\Gamma(a)\Gamma(b) & = \int_0^\infty\exp(-t)t^{a-1}t^{b-1}t\mathrm{d}t\int_0^1\mu^{a-1}(1-\mu)^{b-1}\mathrm{d}\mu \\
& = \Gamma(a+b)\int_0^1\mu^{a-1}(1-\mu)^{b-1}\mathrm{d}\mu
\end{aligned}


# Circular Bump Function

```julia
using Plots

t = collect(-1:0.01:1)
cb(t) = abs(t)<1/2 ? sqrt.(0.25 .- t.^2) : 0
p = plot(t,cb.(t), xlim=[-1,1], ylim=[-0.1,1], label=:false, linewidth=4, xlabel="t")
```

# Fourier transform

Considering that the signal is read and even:

$$
\begin{align}
y(t) &= \frac{a_0}{2} + \sum_{n=1}^{\infty} a_n \frac{cos(n\pi t)}{L} \\
\end{align}
$$

The following integral is derived with Mathematica online calculator.

$$
\begin{align}
a(0) &= \int_{-L}^{+L} y(t) dt  \\
&= \int_{-0.5}^{+0.5} \sqrt{\frac{1}{4}-t^2} dt \\
&= 0.392699
\end{align}
$$

And,

$$
\begin{align}
a(n) &= \frac{1}{L} \int_{-L}^{+L} y(t) cos(\frac{n \pi t}{L}) dt  \\
&= \int_{-0.5}^{+0.5} \sqrt{\frac{1}{4}-t^2} cos(n \pi t) dt \\
&= \frac{J_1(\frac{n\pi}{2})}{2n}
\end{align}
$$

## M program

```julia
using SpecialFunctions

function Mp(m, t)
    a0 = 0.392699
    y = (a0/2)*ones(size(t))
    for n=1:m
        an = besselj(1, n*π/2)/(2*n)
        y += an*cos.(n*π*t)
    end
    y
end
```

    Mp (generic function with 1 method)

```julia
m = [0, 1, 5, 10, 25, 50]

for k in m
    y = Mp(k, t)
    p = plot!(p, t,y, linewidth=3, label="m="*string(k), title="Fourier series for differnt M")
end

display(p)
```

## RMS error

```julia
function err(m)
    t = collect(-1:0.01:1)
    sqrt(sum((Mp(m, t)-cb.(t)).^2)/size(t,1))
end

plot(m,err.(m), seriestype=:scatter, xlabel="Number of terms in Fourier sum"
    , ylabel="RMS error", title="RMS error", label=:false)
```

## Adjourn

```julia
using Dates
println("mahdiar")
Dates.format(now(), "Y/U/d HH:MM")
```

    mahdiar

    "2021/February/4 16:53"


```python
# This cell just imports relevant modules

import numpy
import pylab
from sympy import sin, cos, exp, ln, Function, Symbol, diff, integrate, limit, oo, series, factorial
from math import pi
import mpmath
%matplotlib inline
import matplotlib.pyplot as plt
```

```python
###### SEQUENCES ######
###### Lecture 6, slide 10 ######
# Finite sequence example. The elements of the sequence are stored in a list
finite_sequence = [2*k for k in range(1, 5)]
# Remember: range(A,B) generates integers from A up to B-1, so we need to use B=5 here
```

```python
###### CONVERGENCE OF SEQUENCES ######
###### Lecture 6, slide 11, 13, 14 ######
k = Symbol('k')
print("As k->infinity, the sequence {k} tends to: %f" % limit(k, k, oo) )
# The 'oo' here is SymPy's notation for infinity

print("As k->infinity, the sequence {1/k} tends to: %f" % limit(1.0/k, k, oo))

print("As k->infinity, the sequence {exp(1/k)} tends to: %f" % limit(exp(1.0/k), k, oo))

print("As k->infinity, the sequence {(k**3 + 2*k - 4)/(k**3 + 1)} tends to: %f"
      % limit((k**3 + 2*k - 4)/(k**3 + 1), k, oo))
```

```python
###### SERIES ######
###### Lecture 6, slide 15 ######
# Using list comprehension:
print("The sum of 3*k + 1 (from k=0 to k=4) is: %f" % sum([3*k + 1 for k in range(0,5)]))
# Note: we could also use the nsum function (part of the module mpmath):
# import mpmath
# print mpmath.nsum(lambda k: 3*k + 1, [0, 4])

x = 1
print("The sum of (x**k)/(k!) from k=0 to k=4, with x = 1, is: %f" % sum([x**k/factorial(k) for k in range(1,5)]))
```

```python
###### ARITHMETIC PROGRESSION ######
###### Lecture 6, slide 18 ######
print("The sum of 5 + 4*k up to the 11th term (i.e. up to k=10) is: %f" % sum([5 + 4*k for k in range(0,11)]))
```

```python
###### GEOMETRIC PROGRESSION ######
###### Lecture 6, slide 21 ######
print("The sum of 3**k up to the 7th term (i.e. up to k=6) is: %f" % sum([3**k for k in range(0,7)]))
```

```python
###### INFINITE SERIES ######
###### Lecture 6, slide 23, 24, 25 ######
print("The sum of the infinite series sum(1/(2**k)) is: %f" % mpmath.nsum(lambda k: 1/(2**k), [1, mpmath.inf]))
print("The sum of the infinite alternating series sum(((-1)**(k+1))/k) is: %f"
      % mpmath.nsum(lambda k: ((-1)**(k+1))/k, [1, mpmath.inf]))
```

```python
###### RATIO TEST ######
###### Lecture 6, slide 27 ######
# A divergent example
k = Symbol('k')
f = (2**k)/(3*k)
f1 = (2**(k+1))/(3*(k+1))
ratio = f1/f

lim = limit(ratio, k, oo)
print("As k -> infinity, the ratio tends to: %f" % lim)
if(lim < 1.0):
    print("The series converges")
elif(lim > 1.0):
    print("The series diverges")
else:
    print("The series either converges or diverges")

# A converging example
f = (2**k)/(5**k)
f1 = (2**(k+1))/(5**(k+1))
ratio = f1/f
```

```python
###### POWER SERIES ######
###### Lecture 6, slide 30 ######
k = Symbol('k')
x = Symbol('x')

a = 1.0/k
f = a*(x**k)

a1 = 1.0/(k+1)
f1 = a1*(x**(k+1))

ratio = abs(a/a1)
R = limit(ratio, k, oo)
print("The radius of convergence (denoted R) is: %f" % R)

x = 0.5
if(abs(x) < 1):
    print("The series converges for |x| = %f (< R)" % abs(x))
elif(abs(x) > 1):
    print("The series diverges for |x| = %f (> R)" % abs(x))
else:
    print("The series either converges or diverges for |x| = %f (== R)\n" % abs(x))
```

```python
###### USEFUL SERIES ######
###### Lecture 6, slide 34 ######
x = Symbol('x')
r = Symbol('r')

# Note: the optional argument 'n' allows us to truncate the series
# after a certain order of x has been reached.
print("1/(1+x) = ", series(1.0/(1.0+x), x, n=4))
print("1/(1-x) = ", series(1.0/(1.0-x), x, n=4))
print("ln(1+x) = ", series(ln(1.0+x), x, n=4))
print("exp(x) = ", series(exp(x), x, n=4))
print("cos(x) = ", series(cos(x), x, n=7))
print("sin(x) = ", series(sin(x), x, n=8))
```

```python
### plot taylor series for ln(1+x) with different number of terms n

def ln_taylor(x, n):
    y = 0
    for i in range(1,n):
        y += (-1)**(i+1) * (x**i)/(factorial(i))
    return y

x = numpy.array([0.01*i for i in range(-50, 50)])

ln = numpy.log(1+x)

n = numpy.array([i for i in range(1, 6)])

y_ln = {}

for i in range(len(n)):
    y_ln['n='+str(n[i])] = []
    for j in range(len(x)):
        y_ln['n='+str(n[i])].append(ln_taylor(x[j], n[i]))

colour = ['c', 'b', 'g', 'y', 'r']

plt.figure(figsize=(9, 9))
for i in range(len(n)):
    plt.plot(x, y_ln['n='+str(n[i])], colour[i], label='n=%.d' % (n[i]))
plt.plot(x, ln, 'k', label='y=ln(1+x)')
plt.xlabel('x')
plt.ylabel('y')
plt.title('Compare function ln(1+x) with taylor series containing different number of terms', fontsize=14)
plt.legend(loc='best', fontsize=14)
plt.grid(True)
plt.show()
```

```python
### plot taylor series for exp(x) with different number of terms n

def exp_taylor(x, n):
    y = 0
    for i in range(n):
        y += x**i / factorial(i)
    return y

x = numpy.array([0.01*i for i in range(0, 500)])

expx = numpy.exp(x)

n = numpy.array([i for i in range(1, 6)])

y_exp = {}

for i in range(len(n)):
    y_exp['n='+str(n[i])] = []
    for j in range(len(x)):
        y_exp['n='+str(n[i])].append(exp_taylor(x[j], n[i]))

colour = ['c', 'g', 'b', 'y', 'r']

plt.figure(figsize=(7, 7))
for i in range(len(n)):
    plt.plot(x, y_exp['n='+str(n[i])], colour[i], label='n=%.d' % (n[i]))
plt.plot(x, expx, 'k', label='y=exp(x)')
plt.xlabel('x')
plt.ylabel('y')
plt.title('Compare function exp(x) with taylor series containing different number of terms', fontsize=14)
plt.legend(loc='best', fontsize=14)
plt.grid(True)
plt.show()
```

```python
### plot taylor series for sin(x) with different number of terms n

def sin_taylor(x, n):
    y = 0
    for i in range(n):
        y += (-1)**(i)*(x**(2*i+1) / factorial(2*i+1))
    return y

x = numpy.array([0.01*i for i in range(-500, 500)])

sinx = numpy.sin(x)

n = numpy.array([i for i in range(1, 6)])

y_sin = {}

for i in range(len(n)):
    y_sin['n='+str(n[i])] = []
    for j in range(len(x)):
        y_sin['n='+str(n[i])].append(sin_taylor(x[j], n[i]))

colour = ['r', 'y', 'g', 'b', 'c']

plt.figure(figsize=(9, 9))
for i in range(len(n)):
    plt.plot(x, y_sin['n='+str(n[i])], colour[i], label='n=%.d' % (n[i]))
plt.plot(x, sinx, 'k', label='y=sin(x)')
plt.xlabel('x')
plt.ylabel('y')
plt.title('Compare function sin(x) with taylor series containing different number of terms', fontsize=14)
plt.legend(loc='best', fontsize=14)
plt.grid(True)
plt.show()
```