```python
%matplotlib inline
```

```python
from IPython.core.display import HTML
css_file = './inet.css'
HTML(open(css_file, "r").read())
```

```python
import matplotlib.pyplot as plt
from matplotlib import collections  as mc
plt.xkcd()

import numpy as np
import pandas as pd
import seaborn as sn
import statsmodels.formula.api as smf

from IPython.html import widgets
from IPython.display import display

import pypwt
```

    :0: FutureWarning: IPython widgets are experimental and may change in the future.

 (Linear) Regression

Q: What is regression?

A: Drawing a "line" through a scatter of data points in a principled way...

Q: What is linear regression?

A: Drawing a straight "line" through a scatter of data points in a principled way...

Linear regression in practice

 Step 1: Grab some data...

```python
# grab the entire Penn World Tables data from the web...
pwt = pypwt.load_pwt_data()
```

```python
#...this gives us a panel (i.e., two dimensional) data set
pwt
```

    <class 'pandas.core.panel.Panel'>
    Dimensions: 38 (items) x 167 (major_axis) x 62 (minor_axis)
    Items axis: country to delta_k
    Major_axis axis: AGO to ZWE
    Minor_axis axis: 1950-01-01 00:00:00 to 2011-01-01 00:00:00

```python
def labor_supply(data, year="1950-01-01"):
    """
    Labor supply in a given year is the product of number of employed
    persons, 'emp', and the average number of hours worked, 'avh'.

    Parameters
    ----------
    data : pandas.Panel
        The Penn World Tables (PWT) data set.
    year : str (default="1950-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".

    Returns
    -------
    L : pandas.Series
        Effective labor supply in units of employed person-years.

    """
    L = data.minor_xs(year)["emp"] * data.minor_xs(year)["avh"]
    return L

def real_gdp_per_unit_labor(data, year="1950-01-01"):
    """
    Real gross domestic product (GDP) per unit labor is the ratio of
    some measure of real output and some measure of labor supply.

    Parameters
    ----------
    data : pandas.Panel
        The Penn World Tables (PWT) data set.
    year : str (default="1950-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".

    Returns
    -------
    rgdppul : pandas.Series
        Real gdp per unit labor supply.

    """
    rgdppul = data.minor_xs(year)["rgdpo"] / labor_supply(data, year)
    return rgdppul

def growth_rate_real_gdp_per_unit_labor(data, start="1950-01-01", end="2011-01-01"):
    """
    Plot the growth rate of real GDP per unit labor over some time period
    against the level of real GDP per unit labor at the start of the time
    period.

    Parameters
    ----------
    data : pandas.Panel
        The Penn World Tables (PWT) data set.
    start : str (default="1950-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".
    end : str (default="2011-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".

    """
    gr = (np.log(real_gdp_per_unit_labor(data, end)) -
          np.log(real_gdp_per_unit_labor(data, start)))
    return gr

def some_interesting_plot(data, start="1950-01-01", end="2011-01-01"):
    """
    Plot the growth rate of real GDP per unit labor over some time period
    against the level of real GDP per unit labor at the start of the time
    period.

    Parameters
    ----------
    data : pandas.Panel
        The Penn World Tables (PWT) data set.
    start : str (default="1950-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".
    end : str (default="2011-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".

    """
    # create the scatter plot
    fig, ax = plt.subplots(1, 1, figsize=(12,9))
    xs = np.log(real_gdp_per_unit_labor(data, start))
    ys = growth_rate_real_gdp_per_unit_labor(data, start, end)
    ax.scatter(xs, ys, color='k')

    # axis labels, title, etc
    ax.set_xlabel('Log income (per unit labor) in 1960', fontsize=25)
    ax.set_xlim(0.95 * xs.min(), 1.05 * xs.max())
    ax.set_ylabel('Income (per unit labor) growth\n({}-{})'.format(start[:4], end[:4]),
                  fontsize=25)
    ax.set_ylim(1.05 * ys.min(), 1.05 * ys.max())
    ax.set_title('Do poor countries grow faster than rich countries?',
                 fontsize=25, family="serif")
```

Let's focus on a relation between...

1. (log) level of real income
2. subsequent growth rate of income

...note that income is "per unit labor supply"

Step 2: plot your data...

```python
some_interesting_plot(data=pwt, start="1960-01-01", end="2010-01-01")
```

Step 3: draw a (straight!) line through your data...

```python
def another_interesting_plot(data, start, end, intercept=2.5, slope=-0.5):
    """
    Plot the growth rate of real GDP per unit labor over some time period
    against the level of real GDP per unit labor at the start of the time
    period. Then add a regression line associated with given values for the
    intercept and slope.

    Parameters
    ----------
    data : pandas.Panel
        The Penn World Tables (PWT) data set.
    start : str (default="1950-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".
    end : str (default="2011-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".
    intercept: float (defalut=2.5)
        Intercept for the regression line.
    slope : float (default=-0.5)
        Slope for the regression line.

    """
    # create the scatter plot
    fig, ax = plt.subplots(1, 1, figsize=(12,9))
    xs = np.log(real_gdp_per_unit_labor(data, start))
    ys = growth_rate_real_gdp_per_unit_labor(data, start, end)
    ax.scatter(xs, ys, color='k')

    # compute the regression line given params
    grid = np.linspace(0.95 * xs.min(), 1.05 * xs.max(), 1000)
    predicted = lambda x: intercept + slope * x
    yhat, = ax.plot(grid, predicted(grid) , color='b',
                    label=r"$\hat{y}_i=%.2f + %.2fx_i$" %(intercept, slope))

    # axis labels, title, etc
    ax.set_xlabel('Log income (per unit labor) in 1960', fontsize=25)
    ax.set_xlim(0.95 * xs.min(), 1.05 * xs.max())
    ax.set_ylabel('Income (per unit labor) growth\n({}-{})'.format(start[:4], end[:4]),
                  fontsize=25)
    ax.set_ylim(1.05 * ys.min(), 1.05 * ys.max())
    ax.legend(bbox_to_anchor=(1.0, 0.95), prop={'size': 25})

```

Since we are drawing straight lines...

$$y_i = \beta_0 + \beta_1 x_i + \epsilon_i$$

where

\begin{align}
    y_i=& \text{income growth from 1960-2010} \\
    x_i=& \ln (\text{income in 1960})
\end{align}

and $i=\text{AGO},\dots,\text{ZWE}$.

```python
another_interesting_plot(pwt, start="1960-01-01", end="2010-01-01")
```

Step 4: choose some measure of deviation between model and data...

Given our choice of model...

$$ y_i = \beta_0 + \beta_1 x_i + \epsilon_i,\ i=1,\dots,N $$

...focus on the sum of squared errors (SSE):

$$ SSE \equiv \sum_{i=1}^N  \epsilon_i^2 = \sum_{i=1}^N \big(y_i - (\beta_0 + \beta_1x_i)\big)^2 $$

Step 5: choose params to minimize deviation between model and data...

```python
def another_static_plot(data, start, end, intercept=2.5, slope=-0.5):
    """
    Plot the growth rate of real GDP per unit labor over some time period
    against the level of real GDP per unit labor at the start of the time
    period. Then add a regression line associated with given values for the
    intercept and slope.

    Parameters
    ----------
    data : pandas.Panel
        The Penn World Tables (PWT) data set.
    start : str (default="1950-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".
    end : str (default="2011-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".
    intercept: float (default=2.5)
        Intercept for the regression line.
    slope : float (default=-0.5)
        Slope for the regression line.

    """
    # create the scatter plot
    fig, ax = plt.subplots(1, 1, figsize=(12,9))
    xs = np.log(real_gdp_per_unit_labor(data, start))
    ys = growth_rate_real_gdp_per_unit_labor(data, start, end)
    ax.scatter(xs, ys, color='k')

    # compute and plot the regression line given current params
    predicted = lambda x: intercept + slope * x
    grid = np.linspace(0.95 * xs.min(), 1.05 * xs.max(), 1000)
    yhat, = ax.plot(grid, predicted(grid) , color='b')
    ssr = np.sum((predicted(xs) - ys)**2)

    # add the residuals to the plot
    lines = zip(zip(xs, ys), zip(xs, predicted(xs)))
    lc = mc.LineCollection(lines, colors='r', linewidths=2)
    ax.add_collection(lc)

    # axis labels, title, etc
    ax.set_xlabel('Log income (per unit labor) in 1960', fontsize=25)
    ax.set_xlim(0.95 * xs.min(), 1.05 * xs.max())
    ax.set_ylabel('Income (per unit labor) growth\n({}-{})'.format(start[:4], end[:4]),
                  fontsize=25)
    fig.suptitle(r'Sum of squared errors (SSE) $\equiv \sum_{i=1}^N (y_i - (\beta_0 + \beta_1x_i))^2$',
                 x=0.5, y=1.025, fontsize=25, family="serif")
    fig.legend([yhat, lc], [r"$\hat{y}_i=\hat{\beta}_0 + \hat{\beta}_1x_i$", r"$SSE={0:.4f}$".format(ssr)],
               bbox_to_anchor=(0.8, 0.85), prop={'size': 25})

# create the interactive widget
intercept_widget = widgets.FloatSlider(value=2.5e0, min=0.0, max=5.0, step=5e-2, description=r"$\hat{\beta}_0$")
slope_widget = widgets.FloatSlider(value=-0.5, min=-1.0, max=1.0, step=5e-2, description=r"$\hat{\beta}_1$")
some_interactive_plot = widgets.interactive(another_static_plot,
                                            data=widgets.fixed(pwt),
                                            start=widgets.fixed("1960-01-01"),
                                            end=widgets.fixed("2010-01-01"),
                                            intercept=intercept_widget,
                                            slope=slope_widget,
                                           )
```

Informal parameter estimation...

```python
display(some_interactive_plot)
```

Formal parameter estimation...

Our estimates $\hat{\beta}_0$ and $\hat{\beta}_1$ solve the following optimization problem...

$$ \min_{\beta_0, \beta_1} \sum_{i=1}^N (y_i - (\beta_0 + \beta_1x_i))^2 $$

Taking of derivatives yields a system of first-order conditions...

\begin{align}
    0 =& -2\sum_{i=1}^N (y_i - (\beta_0 + \beta_1x_i)) \\
    0 =& -2\sum_{i=1}^N (y_i - (\beta_0 + \beta_1x_i))x_i
\end{align}

...which after some algebra yields...

\begin{align}
    \hat{\beta}_0 =& \bar{y} - \hat{\beta}_1 \bar{x} \\
    \hat{\beta}_1 =& \frac{\frac{1}{N}\sum_{i=1}^N(x_i - \bar{x})(y_i - \bar{y})}{\frac{1}{N}\sum_{i=1}^N(x_i - \bar{x})^2}
\end{align}

Intuition for $\hat{\beta}_1$...

\begin{align}
    \hat{\beta}_1 =& \frac{Cov(x, y)}{Var(x)} \\
    \equiv& \frac{\text{How much do $x$ and $y$ vary together?}}{\text{How much does $x$ vary in general?}}
\end{align}

```python
# compute the optimal parameter estimates "by hand"
estimated_slope = xs.cov(ys) / xs.var()
estimated_intercept = ys.mean() - estimated_slope * xs.mean()
```

```python
print "Estimated intercept: {}".format(estimated_intercept)
print "Estimated slope: {}".format(estimated_slope)
```

    Estimated intercept: 2.60212926608
    Estimated slope: -0.610771408878

```python
# define the variables
xs = np.log(real_gdp_per_unit_labor(pwt, '1960-01-01'))
ys = growth_rate_real_gdp_per_unit_labor(pwt, '1960-01-01', '2010-01-01')

# compute and plot the regression line given current params
data = pd.DataFrame.from_dict({'x': xs, 'y': ys})
lm = smf.ols(formula="y ~ x", data=data).fit()
```

```python
lm.summary()
```

OLS Regression Results

  Dep. Variable:	            y	          R-squared:         	    0.438

  Model:	                   OLS	         Adj. R-squared:    	    0.420

  Method:	             Least Squares	    F-statistic:       	    24.19

  Date:	             Thu, 21 May 2015	   Prob (F-statistic):	 2.71e-05

  Time:	                 10:50:38	       Log-Likelihood:    	  -21.613

  No. Observations:	          33	        AIC:               	    47.23

  Df Residuals:	              31	        BIC:               	    50.22

  Df Model:	                   1

  Covariance Type:	      nonrobust

      	         coef	     std err	      t	      P>|t|	 [95.0% Conf. Int.]

  Intercept	     2.6021	     0.276	     9.442	  0.000	     2.040     3.164

  x	            -0.6108	     0.124	    -4.919	  0.000	    -0.864    -0.358

  Omnibus:	        6.668	   Durbin-Watson:     	    1.627

  Prob(Omnibus):	  0.036	   Jarque-Bera (JB):  	    5.773

  Skew:	          -1.021	   Prob(JB):          	   0.0558

  Kurtosis:	       3.178	   Cond. No.          	     8.68

```python
def plot_fitted_model(data, start="1950-01-01", end="2011-01-01"):
    """
    Plot the growth rate of real GDP per unit labor over some time period
    against the level of real GDP per unit labor at the start of the time
    period. Then add a regression line associated with given values for the
    intercept and slope.

    Parameters
    ----------
    data : pandas.Panel
        The Penn World Tables (PWT) data set.
    start : str (default="1950-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".
    end : str (default="2011-01-01")
        Some year between 1950 and 2011. Format should be "YYYY-01-01".

    """
    # create the scatter plot
    fig, ax = plt.subplots(1, 1, figsize=(12,9))
    xs = np.log(real_gdp_per_unit_labor(data, start))
    ys = (np.log(real_gdp_per_unit_labor(data, end)) -
          np.log(real_gdp_per_unit_labor(data, start)))
    ax.scatter(xs, ys, color='k')

    # compute and plot the regression line given current params
    data = pd.DataFrame.from_dict({'x': xs, 'y': ys})
    lm = smf.ols(formula="y ~ x", data=data).fit()
    predicted = lambda x: lm.params[0] + lm.params[1] * x

    grid = np.linspace(0.95 * xs.min(), 1.05 * xs.max(), 1000)
    yhat, = ax.plot(grid, predicted(grid) , color='b')
    ssr = np.sum((predicted(xs) - ys)**2)

    # add the residuals to the plot
    lines = zip(zip(xs, ys), zip(xs, predicted(xs)))
    lc = mc.LineCollection(lines, colors='r', linewidths=2)
    ax.add_collection(lc)

    # axis labels, title, etc
    ax.set_xlabel('Log income (per unit labor) in 1960', fontsize=25)
    ax.set_xlim(0.95 * xs.min(), 1.05 * xs.max())
    ax.set_ylabel('Income (per unit labor) growth\n({}-{})'.format(start[:4], end[:4]),
                  fontsize=25)
    ax.set_ylim(1.05 * ys.min(), 1.05 * ys.max())
    fig.legend([yhat, lc], [r"$\hat{y}_i=%.4f + %.4f x_i$" % (lm.params[0], lm.params[1]),
                            r"$SSR={0:.4f}$".format(ssr)],
               bbox_to_anchor=(0.85, 0.85), prop={'size': 25})

```

Plotting the final result...

```python
plot_fitted_model(pwt, '1960-01-01', '2010-01-01')
```

Have we found any evidence for convergence?

Yes...

* slope coefficient has the correct sign

...but...

What about...

* Omitted variable bias!
* Sample selection issues!
* Measurement error?
* etc.


# Building a Spam Filter with Naive Bayes

In this project, we're going to build a spam filter for SMS messages using the multinomial Naive Bayes algorithm. Our goal is to write a program that classifies new messages with an accuracy greater than 80% — so we expect that more than 80% of the new messages will be classified correctly as spam or ham (non-spam).

To train the algorithm, we'll use a dataset of 5,572 SMS messages that are already classified by humans. The dataset was put together by Tiago A. Almeida and José María Gómez Hidalgo, and it can be downloaded from the [The UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/sms+spam+collection). The data collection process is described in more details on [this page](http://www.dt.fee.unicamp.br/~tiago/smsspamcollection/#composition), where you can also find some of the papers authored by Tiago A. Almeida and José María Gómez Hidalgo.

## Exploring the Dataset

We'll now start by reading in the dataset.

```python
import pandas as pd

sms_spam = pd.read_csv('SMSSpamCollection', sep='\t', header=None, names=['Label', 'SMS'])

print(sms_spam.shape)
sms_spam.head()
```

    (5572, 2)

      Label
      SMS

      0
      ham
      Go until jurong point, crazy.. Available only ...

      1
      ham
      Ok lar... Joking wif u oni...

      2
      spam
      Free entry in 2 a wkly comp to win FA Cup fina...

      3
      ham
      U dun say so early hor... U c already then say...

      4
      ham
      Nah I don't think he goes to usf, he lives aro...

Below, we see that about 87% of the messages are ham, and the remaining 13% are spam. This sample looks representative, since in practice most messages that people receive are ham.

```python
sms_spam['Label'].value_counts(normalize=True)
```

    ham     0.865937
    spam    0.134063
    Name: Label, dtype: float64

## Training and Test Set

We're now going to split our dataset into a training and a test set, where the training set accounts for 80% of the data, and the test set for the remaining 20%.

```python
# Randomize the dataset
data_randomized = sms_spam.sample(frac=1, random_state=1)

# Calculate index for split
training_test_index = round(len(data_randomized) * 0.8)

# Training/Test split
training_set = data_randomized[:training_test_index].reset_index(drop=True)
test_set = data_randomized[training_test_index:].reset_index(drop=True)

print(training_set.shape)
print(test_set.shape)
```

    (4458, 2)
    (1114, 2)

We'll now analyze the percentage of spam and ham messages in the training and test sets. We expect the percentages to be close to what we have in the full dataset, where about 87% of the messages are ham, and the remaining 13% are spam.

```python
training_set['Label'].value_counts(normalize=True)
```

    ham     0.86541
    spam    0.13459
    Name: Label, dtype: float64

```python
test_set['Label'].value_counts(normalize=True)
```

    ham     0.868043
    spam    0.131957
    Name: Label, dtype: float64

The results look good! We'll now move on to cleaning the dataset.

## Data Cleaning

To calculate all the probabilities required by the algorithm, we'll first need to perform a bit of data cleaning to bring the data in a format that will allow us to extract easily all the information we need.

Essentially, we want to bring data to this format:

### Letter Case and Punctuation

We'll begin with removing all the punctuation and bringing every letter to lower case.

```python
# Before cleaning
training_set.head()
```

      Label
      SMS

      0
      ham
      Yep, by the pretty sculpture

      1
      ham
      Yes, princess. Are you going to make me moan?

      2
      ham
      Welp apparently he retired

      3
      ham
      Havent.

      4
      ham
      I forgot 2 ask ü all smth.. There's a card on ...

```python
# After cleaning
training_set['SMS'] = training_set['SMS'].str.replace('\W', ' ')
training_set['SMS'] = training_set['SMS'].str.lower()
training_set.head()
```

      Label
      SMS

      0
      ham
      yep  by the pretty sculpture

      1
      ham
      yes  princess  are you going to make me moan

      2
      ham
      welp apparently he retired

      3
      ham
      havent

      4
      ham
      i forgot 2 ask ü all smth   there s a card on ...

### Creating the Vocabulary

Let's now move to creating the vocabulary, which in this context means a list with all the unique words in our training set.

```python
training_set['SMS'] = training_set['SMS'].str.split()

vocabulary = []
for sms in training_set['SMS']:
    for word in sms:
        vocabulary.append(word)

vocabulary = list(set(vocabulary))
```

It looks like there are 7,783 unique words in all the messages of our training set.

```python
len(vocabulary)
```

    7783

### The Final Training Set

We're now going to use the vocabulary we just created to make the data transformation we want.

```python
word_counts_per_sms = {unique_word: [0] * len(training_set['SMS']) for unique_word in vocabulary}

for index, sms in enumerate(training_set['SMS']):
    for word in sms:
        word_counts_per_sms[word][index] += 1
```

```python
word_counts = pd.DataFrame(word_counts_per_sms)
word_counts.head()
```

      user
      repeating
      mob
      exmpel
      145
      basically
      wrkin
      as
      dieting
      contacts
      ...
      simpler
      nalli
      yest
      barkleys
      could
      services
      waaaat
      store
      aha
      okors

      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      ...
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0

      1
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      ...
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0

      2
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      ...
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0

      3
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      ...
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0

      4
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0
      ...
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0

5 rows × 7783 columns

```python
training_set_clean = pd.concat([training_set, word_counts], axis=1)
training_set_clean.head()
```

      Label
      SMS
      ticket
      kappa
      too
      abdomen
      unhappy
      hoody
      start
      die
      ...
      09058095201
      chase
      thru
      ru
      xclusive
      fellow
      red
      entitled
      auto
      bothering

      0
      ham
      [yep, by, the, pretty, sculpture]
      0
      0
      0
      0
      0
      0
      0
      0
      ...
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0

      1
      ham
      [yes, princess, are, you, going, to, make, me,...
      0
      0
      0
      0
      0
      0
      0
      0
      ...
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0

      2
      ham
      [welp, apparently, he, retired]
      0
      0
      0
      0
      0
      0
      0
      0
      ...
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0

      3
      ham
      [havent]
      0
      0
      0
      0
      0
      0
      0
      0
      ...
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0

      4
      ham
      [i, forgot, 2, ask, ü, all, smth, there, s, a,...
      0
      0
      0
      0
      0
      0
      0
      0
      ...
      0
      0
      0
      0
      0
      0
      0
      0
      0
      0

5 rows × 7785 columns

## Calculating Constants First

We're now done with cleaning the training set, and we can begin creating the spam filter. The Naive Bayes algorithm will need to answer these two probability questions to be able to classify new messages:

\begin{equation}
P(Spam | w_1,w_2, ..., w_n) \propto P(Spam) \cdot \prod_{i=1}^{n}P(w_i|Spam)
\end{equation}

\begin{equation}
P(Ham | w_1,w_2, ..., w_n) \propto P(Ham) \cdot \prod_{i=1}^{n}P(w_i|Ham)
\end{equation}

Also, to calculate P(wi|Spam) and P(wi|Ham) inside the formulas above, we'll need to use these equations:

\begin{equation}
P(w_i|Spam) = \frac{N_{w_i|Spam} + \alpha}{N_{Spam} + \alpha \cdot N_{Vocabulary}}
\end{equation}

\begin{equation}
P(w_i|Ham) = \frac{N_{w_i|Ham} + \alpha}{N_{Ham} + \alpha \cdot N_{Vocabulary}}
\end{equation}

Some of the terms in the four equations above will have the same value for every new message. We can calculate the value of these terms once and avoid doing the computations again when a new messages comes in. Below, we'll use our training set to calculate:

- P(Spam) and P(Ham)
- NSpam, NHam, NVocabulary

We'll also use Laplace smoothing and set $\alpha = 1$.

```python
# Isolating spam and ham messages first
spam_messages = training_set_clean[training_set_clean['Label'] == 'spam']
ham_messages = training_set_clean[training_set_clean['Label'] == 'ham']

# P(Spam) and P(Ham)
p_spam = len(spam_messages) / len(training_set_clean)
p_ham = len(ham_messages) / len(training_set_clean)

# N_Spam
n_words_per_spam_message = spam_messages['SMS'].apply(len)
n_spam = n_words_per_spam_message.sum()

# N_Ham
n_words_per_ham_message = ham_messages['SMS'].apply(len)
n_ham = n_words_per_ham_message.sum()

# N_Vocabulary
n_vocabulary = len(vocabulary)

# Laplace smoothing
alpha = 1
```

## Calculating Parameters

Now that we have the constant terms calculated above, we can move on with calculating the parameters $P(w_i|Spam)$ and $P(w_i|Ham)$. Each parameter will thus be a conditional probability value associated with each word in the vocabulary.

The parameters are calculated using the formulas:

\begin{equation}
P(w_i|Spam) = \frac{N_{w_i|Spam} + \alpha}{N_{Spam} + \alpha \cdot N_{Vocabulary}}
\end{equation}

\begin{equation}
P(w_i|Ham) = \frac{N_{w_i|Ham} + \alpha}{N_{Ham} + \alpha \cdot N_{Vocabulary}}
\end{equation}

```python
# Initiate parameters
parameters_spam = {unique_word:0 for unique_word in vocabulary}
parameters_ham = {unique_word:0 for unique_word in vocabulary}

# Calculate parameters
for word in vocabulary:
    n_word_given_spam = spam_messages[word].sum()   # spam_messages already defined in a cell above
    p_word_given_spam = (n_word_given_spam + alpha) / (n_spam + alpha*n_vocabulary)
    parameters_spam[word] = p_word_given_spam

    n_word_given_ham = ham_messages[word].sum()   # ham_messages already defined in a cell above
    p_word_given_ham = (n_word_given_ham + alpha) / (n_ham + alpha*n_vocabulary)
    parameters_ham[word] = p_word_given_ham
```

## Classifying A New Message

Now that we have all our parameters calculated, we can start creating the spam filter. The spam filter can be understood as a function that:

- Takes in as input a new message (w1, w2, ..., wn).
- Calculates P(Spam|w1, w2, ..., wn) and P(Ham|w1, w2, ..., wn).
- Compares the values of P(Spam|w1, w2, ..., wn) and P(Ham|w1, w2, ..., wn), and:
    - If P(Ham|w1, w2, ..., wn) > P(Spam|w1, w2, ..., wn), then the message is classified as ham.
    - If P(Ham|w1, w2, ..., wn) < P(Spam|w1, w2, ..., wn), then the message is classified as spam.
    -  If P(Ham|w1, w2, ..., wn) = P(Spam|w1, w2, ..., wn), then the algorithm may request human help.

```python
import re

def classify(message):
    '''
    message: a string
    '''

    message = re.sub('\W', ' ', message)
    message = message.lower().split()

    p_spam_given_message = p_spam
    p_ham_given_message = p_ham

    for word in message:
        if word in parameters_spam:
            p_spam_given_message *= parameters_spam[word]

        if word in parameters_ham:
            p_ham_given_message *= parameters_ham[word]

    print('P(Spam|message):', p_spam_given_message)
    print('P(Ham|message):', p_ham_given_message)

    if p_ham_given_message > p_spam_given_message:
        print('Label: Ham')
    elif p_ham_given_message < p_spam_given_message:
        print('Label: Spam')
    else:
        print('Equal proabilities, have a human classify this!')
```

```python
classify('WINNER!! This is the secret code to unlock the money: C3421.')
```

    P(Spam|message): 1.3481290211300841e-25
    P(Ham|message): 1.9368049028589875e-27
    Label: Spam

```python
classify("Sounds good, Tom, then see u there")
```

    P(Spam|message): 2.4372375665888117e-25
    P(Ham|message): 3.687530435009238e-21
    Label: Ham

## Measuring the Spam Filter's Accuracy

The two results above look promising, but let's see how well the filter does on our test set, which has 1,114 messages.

We'll start by writing a function that returns classification labels instead of printing them.

```python
def classify_test_set(message):
    '''
    message: a string
    '''

    message = re.sub('\W', ' ', message)
    message = message.lower().split()

    p_spam_given_message = p_spam
    p_ham_given_message = p_ham

    for word in message:
        if word in parameters_spam:
            p_spam_given_message *= parameters_spam[word]

        if word in parameters_ham:
            p_ham_given_message *= parameters_ham[word]

    if p_ham_given_message > p_spam_given_message:
        return 'ham'
    elif p_spam_given_message > p_ham_given_message:
        return 'spam'
    else:
        return 'needs human classification'
```

Now that we have a function that returns labels instead of printing them, we can use it to create a new column in our test set.

```python
test_set['predicted'] = test_set['SMS'].apply(classify_test_set)
test_set.head()
```

      Label
      SMS
      predicted

      0
      ham
      Later i guess. I needa do mcat study too.
      ham

      1
      ham
      But i haf enuff space got like 4 mb...
      ham

      2
      spam
      Had your mobile 10 mths? Update to latest Oran...
      spam

      3
      ham
      All sounds good. Fingers . Makes it difficult ...
      ham

      4
      ham
      All done, all handed in. Don't know if mega sh...
      ham

Now, we'll write a function to measure the accuracy of our spam filter to find out how well our spam filter does.

```python
correct = 0
total = test_set.shape[0]

for row in test_set.iterrows():
    row = row[1]
    if row['Label'] == row['predicted']:
        correct += 1

print('Correct:', correct)
print('Incorrect:', total - correct)
print('Accuracy:', correct/total)
```

    Correct: 1100
    Incorrect: 14
    Accuracy: 0.9874326750448833

The accuracy is close to 98.74%, which is really good. Our spam filter looked at 1,114 messages that it hasn't seen in training, and classified 1,100 correctly.

## Next Steps

In this project, we managed to build a spam filter for SMS messages using the multinomial Naive Bayes algorithm. The filter had an accuracy of 98.74% on the test set we used, which is a pretty good result. Our initial goal was an accuracy of over 80%, and we managed to do way better than that.

Next steps include:

- Analyze the 14 messages that were classified incorrectly and try to figure out why the algorithm classified them incorrectly
- Make the filtering process more complex by making the algorithm sensitive to letter case


 9. Markov chains

 Andrej Košir, Lucami, FE

 Kontakt: prof. dr. Andrej Košir, andrej.kosir@lucami.fe.uni-lj.si, skype=akosir_sid

 1

## ■ Intro

- Goals:
    - Markov chain modeling
    - Markov chain state analysis
- Stochastic process on finite states, most useful and used. We limit to:
    - Discrete times
    - Discrete state space (finite or countably many states)
- Case: user behavior
- Usage
    - Finite state machine analysis: software, TC system, ...
    - Models in economy
- What are questions:
    - Distributions of visits
    - Long term behavior
    - Absorption of states

 2

## ■ Sections

9.1. Definition, basic characteristics

■ Markov chain definition $\large{*}$

■ Transition matrix, distribution of states $\large{*}$

9.2. Classification of states

■ Communication of states and period $\large{*}$

■ Recurrent and transient state $\large{*}$

■ Ergodic Markov chain and return times

■ Stationary (limit) distribution $\large{*}$

■ Finite Markov chain $\large{*}$

9.3. Transition matrix estimation

■ Transition probability estimation

■ Confidence intervals and quantiles

■ Sample size determination

■ Numerical aspect

■ Case: telecommunication service user modeling

 3

## 9.1. Definition, basic characteristics

■ Markov chain definition

■ Transition matrix, distribution of states

 4

## ■ Markov chain definition

- State space $S$, $m=|S|$
    - Finite, countable, continuous

- Time $T$
    - Discrete, continuous

- Stochastic process:
$$ X_n : T \to S $$

- Discrete Markov chain is a discrete time discrete space Markov chain having Markov property
$$ P[X_{n+1}|_{X_n, X_{n-1}, \ldots, X_0}] = P[X_{n+1}|_{X_n}] = P[X_1|_{X_0}] $$

- Transition probability of one step
$$ p_{ij} = P[X_{n+1}=j|_{X_n=i}] $$

- Finite state space means there is a matrix  of transition probabilities
$$ P = [p_{ij}], \; i,j\in S. $$
    - Row sums are equal to 1.

 5

```python
import numpy as np
from sympy import Matrix
from sympy.functions import re

# Case 1
# Transition matrix
P = np.array([[0.7, 0.2, 0.1, 0.0],
              [0.2, 0.5, 0.2, 0.1],
              [0.05, 0.15, 0.6, 0.2],
              [0.0, 0.2, 0.0, 0.8]])
```

## ■ Transition matrix, distribution of states

- Transition matrix for $n$ steps
    - $p_{ij}^{(n)} = P[X_{k+n}|_{X_k=i}]$, $P^{(n)}=[p_{ij}^{(n)}]$
    - From total probability formula
    $$ P^{(n)} = P^n $$

- Distribution of states:
$$ \pi_i^n = P[X_n=i], \pi^n = [\pi_1^n, \ldots, \pi_i^n] $$
    - Initial distribution: $\pi^0$. If initial state $X_0=i_0$ is known
    $$ \pi^0=\delta_{i_0 i} $$
    - From total probability formula:
    $$ \pi^n = P^{(n)} \pi^0 $$
    - More on limit distribution later

 6

## 9.2. Classification of states

■ Communication of states and period

■ Recurrent and transient state

■ Ergodic Markov chain and return times

■ Stationary (limit) distribution

■ Finite Markov chain

 7

## ■ Communication of states and period

- Accessible state
    - State $j$ is accessible from state $i$ if $\exists n: p_{ij}^{(n)}>0$. Notation $i\leftrightarrow j$
    - Two states communicate if $i\rightarrow$ and $i \leftarrow j$, denoted by $i\leftrightarrow j$
    - „Communicate“ is equivalent relation

- Markov chain is irreducible if there are only one single communicating class

- Period of state $i$ ia denoted by $d(i)$ is greatest common divisor of all $n$ such that $P_{ii}^{(n)} > 0$:
    - If $i\leftrightarrow j$ then $d(i) = d(j)$;
    - State is aperiodic if $d(i)=1$;

- State $i$ is zero state if
$$ \lim_{n\to\infty} P_{ii}^{(n)} = 0. $$

 8

```python
# ---------------------------------------------------------------------------------------------------
# Classification of states
print ('Transition matrix P=\n', P)
print ('\nTransition matrix P^2=\n', np.linalg.matrix_power(P, 2))

# All entries of P^2 are non-zero
# => All atates are connected
```

    Transition matrix P=
     [[0.7  0.2  0.1  0.  ]
     [0.2  0.5  0.2  0.1 ]
     [0.05 0.15 0.6  0.2 ]
     [0.   0.2  0.   0.8 ]]

    Transition matrix P^2=
     [[0.535 0.255 0.17  0.04 ]
     [0.25  0.34  0.24  0.17 ]
     [0.095 0.215 0.395 0.295]
     [0.04  0.26  0.04  0.66 ]]

```python
# ---------------------------------------------------------------------------------------------------
# Decompose transition matrix
# Use Jordan decomposition
#mP = Matrix(P)
#mB, mJ = mP.jordan_form()
#B, J = np.array(mB), np.array(mJ)
B = np.array([[-0.5,-0.776313,0.659526,-0.413722],
              [-0.5,-0.212847,-0.11514,0.830469],
              [-0.5,0.283275,-0.73692,-0.123153],
              [-0.5,0.521335,0.0933625,-0.352121]])
J = np.array([[1.,0,0,0],
              [0,0.718345,0,0],
              [0,0,0.553349,0],
              [0,0,0,0.328305]])

# Test decomposition
err_mat = P - np.dot(B, np.dot(J, np.linalg.inv(B)))
print ('\nDecomposition error norm = ', np.linalg.norm(err_mat))
```

    Decomposition error norm =  7.249920892047682e-07

## ■ Recurrent and transient state

- Probability of return after exactly $n$ steps
$$ f_{ij}^n = P[X_n=j|_{X_{n-1}\ne j, \ldots, X_1\ne j, X_0=i}] $$

- Random variable of transitions from $i$ to $j$ is denoted by $T_{i}$ and its distribtion is
$$ T_{ij} \sim \left(\matrix{0 & 1 & 2 & \ldots \cr f_{ij}^0 & f_{ij}^1 & f_{ij}^2 & \ldots}\right) $$
    - Denote $T_i = T_{ii}$
    - Denote $\sum_{n=0}^\infty f_{ii}^n = f_{ii}^*$. Value
    $$ 1 - f_{ii}^* $$
    is a probability that Markov Chain newer returns to state 𝑖 after leaving it.

- State $i$ is **recurrent** if $f_{ii}^* = 1$. State is **transient** if it is not recurrent.
    - Recurrence and transience are communicating class properties

- We have: state $i$ is recurrent if and only if
$$ \sum_{n=1}^\infty P_{ii}^{(n)} = \infty $$

- Also: if $i\leftrightarrow j$ then both states are either transient or recurrent

 9

```python
# Recurent, transient?

# Eigenvalues
eigVals, eigVecs = np.linalg.eig(P.T)

# Get it sorted
idx = eigVals.argsort()[::-1]
eigVa = eigVals[idx]
eigVc = eigVecs[:,idx]

las_bf =  (1./(1-eigVa[1:4]))-1
print ('las_bf', las_bf)

sumJn = np.diag(np.insert(las_bf, 0, np.inf))
print ('\nSum J^n\n', sumJn)

sumPn = np.dot(B, np.dot(sumJn, np.linalg.inv(B)))
print ('\nSum P^n\n', sumPn)

# Since all diagonal elements of sum P^n are infinity
#  => all states are recurrent
```

    las_bf [2.55044912 1.23888596 0.48877142]

    Sum J^n
    [[       inf 0.         0.         0.        ]
     [0.         2.55044912 0.         0.        ]
     [0.         0.         1.23888596 0.        ]
     [0.         0.         0.         0.48877142]]

    Sum P^n
    [[inf inf inf inf]
     [inf inf inf inf]
     [inf inf inf inf]
     [inf inf inf inf]]

```python
def inf_sum(a_lst):
    return [(la/(1.0-la) if la < 1 else np.infty) for la in a_lst]

# ---------------------------------------------------------------------------------------------------
# Are states recurrent?
# Compute sum of P^n
sumJ = np.diag(inf_sum(np.diag(J)))
sumP = np.dot(B, np.dot(sumJ, np.linalg.inv(B)))

print ('\nSum of P^n = \n', sumP)
# => All states are reccurent
```

    Sum of P^n =
    [[inf inf inf inf]
     [inf inf inf inf]
     [inf inf inf inf]
     [inf inf inf inf]]

## ■ Ergodic Markov chain and return times

- Markov chain is **Ergodic** if it is
    - Irreducible
    - Recurrent (all states are recurrent)
    - Aperiodic (all states are aperiodic)
    - Nonzero (all states are positive)

- Expected return time:
$$ E(T_i) = \sum_{n=0}^\infty n f_{ii}^{(n)} $$

- In ergodic Markov chain
$$ \lim_{n\to\infty} P_{ii}^{(n)} = \frac{1}{E(T_i)} $$
$$ \lim_{n\to\infty} P_{ij}^{(n)} = \lim_{n\to\infty} P_{ii}^{(n)} $$

- Recurrent **zero** state $i$:
$$ \lim_{n\to\infty} P_{ii}^{(n)} = 0 \; \Leftrightarrow \;  E(T_i) = \infty  $$

- Recurrent **positive** = **ergodic** state $i$:
$$ \lim_{n\to\infty} P_{ii}^{(n)} > 0 \; \Leftrightarrow \; E(T_i) < \infty  $$

 10

```python
def inf_lim(a_lst):
    return [(0 if la < 1 else 1) for la in a_lst]

# ---------------------------------------------------------------------------------------------------
# Zero or positive states?
# Compute lim of P^n
limJ = np.diag(inf_lim(np.diag(J)))
limP = np.dot(B, np.dot(limJ, np.linalg.inv(B)))

print ('\nlim of P^n = \n', limP)

# Since all lim P^n are positive
# => all states are positive

# Expected times of visits
ETi = 1.0/np.diag(limP)

print ('\nExpected number of visits py states:\n', ETi)
```

    lim of P^n =
    [[0.21301781 0.27218919 0.18934915 0.32544385]
     [0.21301781 0.27218919 0.18934915 0.32544385]
     [0.21301781 0.27218919 0.18934915 0.32544385]
     [0.21301781 0.27218919 0.18934915 0.32544385]]

    Expected number of visits py states:
    [4.69444311 3.67391519 5.28124903 3.07272668]

## ■ Stationary (limit) distribution

- State distribution after $n$ steps
$$ \pi_i^n = P[X_n=i], \qquad \pi^n = [\pi_1^n, \ldots, \pi_i^n] $$

- Stationary (limit) distribution:
  $$ \pi = \lim_{n\to\infty} \pi^n $$
    - When uniquely defined: when all states are recurrent positive

- Ergodic MC: stationary distribution exists
- Stationary distribution of ergodic MC chain is left eigenvector of transition matrix:
  $$ \pi^\top = \pi^\top P. $$

- Note: in an ergodic MC a stationary distribution independent of initial distribution $\pi^0$.

 11

```python
# ---------------------------------------------------------------------------------------------------
# Limit distribuion
eigVals, eigVecs = np.linalg.eig(P.T)
pi = eigVecs[:, 0]/sum(eigVecs[:, 0])

print ('\neigVals=\n', eigVals)
#print '\neigVecs=\n', eigVecs
print ('\nLimit distribution =', pi)
```

    eigVals=
    [1.         0.32830522 0.71834549 0.55334929]

    Limit distribution = [0.21301775 0.27218935 0.18934911 0.32544379]

## ■ Probability of absorption

- Absorption: chain never leave a subset of states $S_0 \subset S$

- What is the probability of absorption?

- If a chain starts in transient state, then either:
    1. It leaves a set of transient states with probability of 1
    2. It enters a set of recurrent states with probability of 1

- Absorbing states $i$: the only state of an absorbing state
We have: $i$ absorbing if and only if
$P_{ii}=1$

- Let $i\in S$ be a transient state, $S_e$ a set of recurrent states:

Probability of absorption: $\pi_i(S_e)$ is the probability MC with $X_0=i$ gets absorbed into the set of states $S_e$

- We have
$$ \lim_{n\to\infty} P_{ij}^{(n)} = \pi_i(S_e)\pi_j. $$

 12

## ■ Finite Markov chain

- Finite number of states $|S|=m < \infty$.

- Each finite MC has recurrent states!

- All recurrent states are positive, that is ergodic and has finite expected return time
$$ \lim_{n\to\infty} P_{ii}^{(n)} > 0. $$

- Classification of states: all states are $S=S_m\cup S_e$, where
    - Transient states: $S_m$
    - Ergodic states: $S_e$

- Transient matrix is of a block form:
    - Ergodic states: $1, \ldots, m-K$
    - Transient state: $m-K+1, \ldots, m$
    $$ P = \left[\matrix{S & 0 \cr R & Q}\right] $$

- Fundamental matrix:
$$ N = (I - Q)^{-1}.  $$

 13

```python
# Case 2
c2P = np.array(
    [[0.7, 0.3, 0.0, 0.0],
     [0.2, 0.8, 0.0, 0.0],
     [0.05, 0.15, 0.6, 0.2],
     [0.0, 0.2, 0.0, 0.8]])

print ('Transition matrix P=\n', c2P)

```

    Transition matrix P=
     [[0.7  0.3  0.   0.  ]
     [0.2  0.8  0.   0.  ]
     [0.05 0.15 0.6  0.2 ]
     [0.   0.2  0.   0.8 ]]

```python
# ---------------------------------------------------------------------------------------------------
# Classification of states
print ('Transition matrix P=\n', c2P)
print ('\nTransition matrix P^2=\n', np.linalg.matrix_power(c2P, 2))
print ('\nTransition matrix P^3=\n', np.linalg.matrix_power(c2P, 3))

# Matrix multiplications by blocks
# => 2 x 2 block of zeros stayes the same
# => There are two connected classes of states
```

    Transition matrix P=
     [[0.7  0.3  0.   0.  ]
     [0.2  0.8  0.   0.  ]
     [0.05 0.15 0.6  0.2 ]
     [0.   0.2  0.   0.8 ]]

    Transition matrix P^2=
     [[0.55  0.45  0.    0.   ]
     [0.3   0.7   0.    0.   ]
     [0.095 0.265 0.36  0.28 ]
     [0.04  0.32  0.    0.64 ]]

    Transition matrix P^3=
     [[0.475  0.525  0.     0.    ]
     [0.35   0.65   0.     0.    ]
     [0.1375 0.3505 0.216  0.296 ]
     [0.092  0.396  0.     0.512 ]]

```python
# ---------------------------------------------------------------------------------------------------
# Limit distribuion
eigVals, eigVecs = np.linalg.eig(c2P.T)

# Get it sorted
idx = eigVals.argsort()[::-1]
eigenValues = eigVals[idx]
eigenVectors = eigVecs[:,idx]

# Normalize eigenvector
pi = eigenVectors[:, 0]/sum(eigenVectors[:, 0])

print ('\neigVals=\n', eigenValues)
#print '\neigVecs=\n', eigenVectors
print ('\nLimit distribution =', pi)
```

    eigVals=
     [1.  0.8 0.6 0.5]

    Limit distribution = [ 0.4  0.6 -0.  -0. ]

```python
# ---------------------------------------------------------------------------------------------------
# Decompose transition matrix
# Use Jordan decomposition
#mP = Matrix(c2P)
#mB, mJ = mP.jordan_form()
#B, J = np.array(mB), np.array(mJ)
c2B = np.array([[1.0, 0, 0, 9.0/4],
              [1.0, 0, 0, -3.0/2],
              [1.0, 1.0, 1.0, -7.0/8],
              [1.0, 1.0, 0, 1.0]])
c2J = np.array([[1.0,0,0,0],
              [0,4.0/5,0,0],
              [0,0,3.0/5,0],
              [0,0,0,1.0/2]])

# Test decomposition
err_mat = c2P - np.dot(c2B, np.dot(c2J, np.linalg.inv(c2B)))
print ('\nDecomposition error norm = ', np.linalg.norm(err_mat))
```

    Decomposition error norm =  1.9675159943996247e-16

```python
def prd(a,b):
    if np.isinf(a) and b == 0:
        return 0
    if np.isinf(b) and a == 0:
        return 0
    return a*b

def myDot(A,B):
    nA1, nA2, nB2 = A.shape[0], A.shape[1], B.shape[1]
    AB = np.zeros((nA1,nB2))
    for i in range(nA1):
        for j in range(nB2):
            curr = 0
            for k in range(nA2):
                curr += prd(A[i,k], B[k,j])
            AB[i, j] = curr
    return AB

# Classification of states
# Recurent, transient?

# Eigenvalues
eigVals, eigVecs = np.linalg.eig(c2P.T)

# Get it sorted
idx = eigVals.argsort()[::-1]
eigVa = eigVals[idx]
eigVc = eigVecs[:,idx]

las_bf =  (1./(1-eigVa[1:4]))-1
sumJn = np.diag(np.insert(las_bf, 0, np.inf))

sumPn = myDot(c2B, myDot(sumJn, np.linalg.inv(c2B)))
print ('\nSum P^n\n', sumPn)

# Since diagonal elements of sum P^n i=1 and i=2 are infinity
#  => states i=1 and i=2 are recurrent
# Since diagonal elements of sum P^n i=3 and i=4 are finite
#  => states i=3 and i=4 are transient

```

    Sum P^n
     [[inf inf 0.  0. ]
     [inf inf 0.  0. ]
     [inf inf 1.5 2.5]
     [inf inf 0.  4. ]]

```python
def inf_lim(a_lst):
    return [(0 if la < 1 else 1) for la in a_lst]

# ---------------------------------------------------------------------------------------------------
# Zero or positive states?
# Compute lim of P^n
limJ = np.diag(inf_lim(np.diag(c2J)))
limP = np.dot(c2B, np.dot(limJ, np.linalg.inv(c2B)))

print ('\nlim of P^n = \n', limP)

# Since all lim P^n are positive
# => all states are positive

# Expected times of visits
ETi = 1.0/np.diag(limP)

print ('\nExpected number of visits py states:\n', ETi)

```

    lim of P^n =
     [[0.4 0.6 0.  0. ]
     [0.4 0.6 0.  0. ]
     [0.4 0.6 0.  0. ]
     [0.4 0.6 0.  0. ]]

    Expected number of visits py states:
     [2.5        1.66666667        inf        inf]

    /home/nbuser/anaconda3_501/lib/python3.6/site-packages/ipykernel/__main__.py:16: RuntimeWarning: divide by zero encountered in true_divide

```python
# ---------------------------------------------------------------------------------------------------
# Limit distribuion
eigVals, eigVecs = np.linalg.eig(c2P.T)

# Get it sorted
idx = eigVals.argsort()[::-1]
eigenValues = eigVals[idx]
eigenVectors = eigVecs[:,idx]

# Normalize eigenvector
pi = eigenVectors[:, 0]/sum(eigenVectors[:, 0])

print ('\neigVals=\n', eigenValues)
#print '\neigVecs=\n', eigenVectors
print ('\nLimit distribution =', pi)
```

    eigVals=
    [1.  0.8 0.6 0.5]

    Limit distribution = [ 0.4  0.6 -0.  -0. ]

## ■ Number of visits of a transient state

- Random variable $n_i$: number of visits of a state $i$ in an arbitrary long time

- Expected number of visits of a state $j$ if initial state is $𝑖$:
 $$ E_i[n_j] < \infty $$

- We have: if state $i$ is transient:
$$ E_i[n_j] = \sum_{k=1}^\infty P_{ij}^{(k)} $$

- Also - fundamental matrix gives expected values:
$$ \left[E_i[n_j]\right] = N $$

 14

```python
# Expected number of visits of recurent states
m = 4 # |S|
K = 2 # number of recurent states

Q = c2P[K:m, K:m]
print ('Part of transition matrix Q = \n', Q)

N = np.linalg.inv(np.eye(m-K)-Q)
print ('\nFundamental matrix N =\n', N)
```

    Part of transition matrix Q =
    [[0.6 0.2]
     [0.  0.8]]

    Fundamental matrix N =
    [[2.5 2.5]
     [0.  5. ]]

## 9.3 Transition matrix estimation

■ Transition probability estimation

■ Confidence intervals and quantiles

■ Sample size determination

■ Numerical aspect

■ Case: telecommunication service user modeling

 15

## ■ Transition probability estimation

- Transition probability
$$ p_{ij} = \frac{n_k}{n} $$
is a statistical estimation;

- Parameter estimation
    - estimate and termine
        - expected value $\overline{x}$
        - standard deviation $\sigma$
        - for a selected risk level $\alpha$ compute $z_\alpha$ (if normal distribution $z_{0.05}=1.96$)
    - Confidence interval:
$$ CI = [\overline{x} - z_\alpha\frac{\sigma}{\sqrt{n}}, \overline{x} + z_\alpha\frac{\sigma}{\sqrt{n}}] $$
    - Sample size: determined from a preset confidence interval length
    $$ |CI| = 2 z_\alpha\frac{\sigma}{\sqrt{n}} $$

 16

## ■ Confidence intervals and quantiles

- Confidence interval visualization

 17

## ■ Transition matrix estimation sample size determination

- From confidence interval size

- Confidence interval for risk level $\alpha=0.05$ ($z_\alpha=1.96$)
$$ CI = [p_0 - z_\alpha\sqrt{p_0(1-p_0)/n}, p_0 + z_\alpha\sqrt{p_0(1-p_0)/n}] $$

For $p=0.65$ and $|CI|=\Delta p$ at $\alpha=0.05$
$$ n\geq \frac{4 z_\alpha^2}{\Delta p^2} p_0 (1-p_0) $$
we get
 - for $\Delta p=0.01:$ $n\geq 34959$,
 - for $\Delta p=0.02:$ $n\geq 175$,
 - for $\Delta p=0.04:$ $n\geq 88$.

 18

## ■ Numerical aspect of transition matrix computation

- We need matrix power
$$ P^n = P^{(n)} $$

- Jordan decomposition
$$ P = B J B^{-1},  $$
where $J$ is Jordan canonical form.

- For each class of states we have
$$ J = \lambda I + R $$
and
$$ P^n = B J^n B^{-1} $$

 19

## ■ Case: telecommunication service user modeling

- A set of telecommunication users $U$

- We split them into classes: $U=U_1 \cup U_2 \cup U_3 \cup U_4$
    - Very satisfied: $U_1$
    - Satisfied: $U_2$
    - Not satisfied: $U_3$
    - Churners: $U_4$

- Initial distribution: estimation of users

- Results:
    - Recurrent - transients states
    - Stationary distribution

 20

## ■ Conclusion

- Most useful: finite Markov chain
- Check the transition matrix assumption: stationarity

 21


```python
%matplotlib inline
```

nn.Transformer 와 TorchText 로 시퀀스-투-시퀀스(Sequence-to-Sequence) 모델링하기
=================================================================================

이 튜토리얼에서는
`nn.Transformer <https://pytorch.org/docs/stable/generated/torch.nn.Transformer.html>`__ 모듈을
이용하는 시퀀스-투-시퀀스(Sequence-to-Sequence) 모델을 학습하는 방법을 배워보겠습니다.

PyTorch 1.2 버젼에는 `Attention is All You Need <https://arxiv.org/pdf/1706.03762.pdf>`__ 논문에
기반한 표준 트랜스포머(transformer) 모듈을 포함하고 있습니다.
트랜스포머 모델은 다양한 시퀀스-투-시퀀스 문제들에서 더 병렬화(parallelizable)가 가능하면서도
순환 신경망(RNN; Recurrent Neural Network)과 비교하여 더 나은 성능을 보임이 입증되었습니다.
``nn.Transformer`` 모듈은 입력(input) 과 출력(output) 사이의 전역적인 의존성(global dependencies)
을 나타내기 위하여 (`nn.MultiheadAttention <https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html>`__ 으로
구현된) 어텐션(attention) 메커니즘에 전적으로 의존합니다.
현재 ``nn.Transformer`` 모듈은 모듈화가 잘 되어 있어
단일 컴포넌트 (예. `nn.TransformerEncoder <https://pytorch.org/docs/stable/generated/torch.nn.TransformerEncoder.html>`__ )
로 쉽게 적용 및 구성할 수 있습니다.

모델 정의하기
----------------

이 튜토리얼에서, 우리는 ``nn.TransformerEncoder`` 모델을 언어 모델링(language modeling) 과제에 대해서 학습시킬 것입니다.
언어 모델링 과제는 주어진 단어 (또는 단어의 시퀀스) 가 다음에 이어지는 단어 시퀀스를 따를 가능성(likelihood)에 대한 확률을 할당하는 것입니다.
먼저, 토큰(token) 들의 시퀀스가 임베딩(embedding) 레이어로 전달되며, 이어서 포지셔널 인코딩(positional encoding) 레이어가 각 단어의 순서를 설명합니다.
(더 자세한 설명은 다음 단락을 참고해주세요.)
``nn.TransformerEncoder`` 는 여러 개의
`nn.TransformerEncoderLayer <https://pytorch.org/docs/stable/generated/torch.nn.TransformerEncoderLayer.html>`__
레이어로 구성되어 있습니다.
``nn.TransformerEncoder`` 내부의 셀프-어텐션(self-attention) 레이어들은 시퀀스 안에서의 이전 포지션에만 집중하도록 허용되기 때문에,
입력(input) 순서와 함께, 정사각 형태의 어텐션 마스크(attention mask) 가 필요합니다.
언어 모델링 과제를 위해서, 미래의 포지션에 있는 모든 토큰들은 마스킹 되어야(가려져야) 합니다.
실제 단어를 얻기 위해서, ``nn.TransformerEncoder`` 의 출력은 로그-소프트맥스(log-Softmax) 로 이어지는 최종 선형(Linear) 레이어로 전달 됩니다.

```python
import math
from typing import Tuple

import torch
from torch import nn, Tensor
import torch.nn.functional as F
from torch.nn import TransformerEncoder, TransformerEncoderLayer
from torch.utils.data import dataset

class TransformerModel(nn.Module):

    def __init__(self, ntoken: int, d_model: int, nhead: int, d_hid: int,
                 nlayers: int, dropout: float = 0.5):
        super().__init__()
        self.model_type = 'Transformer'
        self.pos_encoder = PositionalEncoding(d_model, dropout)
        encoder_layers = TransformerEncoderLayer(d_model, nhead, d_hid, dropout)
        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
        self.encoder = nn.Embedding(ntoken, d_model)
        self.d_model = d_model
        self.decoder = nn.Linear(d_model, ntoken)

        self.init_weights()

    def init_weights(self) -> None:
        initrange = 0.1
        self.encoder.weight.data.uniform_(-initrange, initrange)
        self.decoder.bias.data.zero_()
        self.decoder.weight.data.uniform_(-initrange, initrange)

    def forward(self, src: Tensor, src_mask: Tensor) -> Tensor:
        """
        Args:
            src: Tensor, shape [seq_len, batch_size]
            src_mask: Tensor, shape [seq_len, seq_len]

        Returns:
            output Tensor of shape [seq_len, batch_size, ntoken]
        """
        src = self.encoder(src) * math.sqrt(self.d_model)
        src = self.pos_encoder(src)
        output = self.transformer_encoder(src, src_mask)
        output = self.decoder(output)
        return output

def generate_square_subsequent_mask(sz: int) -> Tensor:
    """Generates an upper-triangular matrix of -inf, with zeros on diag."""
    return torch.triu(torch.ones(sz, sz) * float('-inf'), diagonal=1)
```

``PositionalEncoding`` 모듈은 시퀀스 안에서 토큰의 상대적인 또는 절대적인 포지션에 대한 어떤 정보를 주입합니다.
포지셔널 인코딩은 임베딩과 합칠 수 있도록 똑같은 차원을 가집니다.
여기에서, 우리는 다른 주파수(frequency) 의 ``sine`` 과 ``cosine`` 함수를 사용합니다.

```python
class PositionalEncoding(nn.Module):

    def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000):
        super().__init__()
        self.dropout = nn.Dropout(p=dropout)

        position = torch.arange(max_len).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))
        pe = torch.zeros(max_len, 1, d_model)
        pe[:, 0, 0::2] = torch.sin(position * div_term)
        pe[:, 0, 1::2] = torch.cos(position * div_term)
        self.register_buffer('pe', pe)

    def forward(self, x: Tensor) -> Tensor:
        """
        Args:
            x: Tensor, shape [seq_len, batch_size, embedding_dim]
        """
        x = x + self.pe[:x.size(0)]
        return self.dropout(x)
```

데이터 로드하고 배치 만들기
-----------------------------

이 튜토리얼에서는 ``torchtext`` 를 사용하여 Wikitext-2 데이터셋을 생성합니다.
torchtext 데이터셋에 접근하기 전에, https://github.com/pytorch/data 을 참고하여 torchdata를 설치하시기 바랍니다.
단어 오브젝트는 훈련 데이터셋(train dataset) 에 의하여 만들어지고, 토큰(token)을 텐서(tensor)로 수치화하는데 사용됩니다.
Wikitext-2에서 보기 드믄 토큰(rare token)은 `<unk>` 로 표현됩니다.

주어진 1D 벡터의 시퀀스 데이터에서, ``batchify()`` 함수는 데이터를 ``batch_size`` 컬럼들로 정렬합니다.
만약 데이터가 ``batch_size`` 컬럼으로 나누어 떨어지지 않으면, 데이터를 잘라내서 맞춥니다.
예를 들어 (총 길이 26의) 알파벳을 데이터로 보고 ``batch_size=4`` 일 때, 알파벳은 길이가 6인 4개의 시퀀스로 나눠집니다:

\begin{align}\begin{bmatrix}
  \text{A} & \text{B} & \text{C} & \ldots & \text{X} & \text{Y} & \text{Z}
  \end{bmatrix}
  \Rightarrow
  \begin{bmatrix}
  \begin{bmatrix}\text{A} \\ \text{B} \\ \text{C} \\ \text{D} \\ \text{E} \\ \text{F}\end{bmatrix} &
  \begin{bmatrix}\text{G} \\ \text{H} \\ \text{I} \\ \text{J} \\ \text{K} \\ \text{L}\end{bmatrix} &
  \begin{bmatrix}\text{M} \\ \text{N} \\ \text{O} \\ \text{P} \\ \text{Q} \\ \text{R}\end{bmatrix} &
  \begin{bmatrix}\text{S} \\ \text{T} \\ \text{U} \\ \text{V} \\ \text{W} \\ \text{X}\end{bmatrix}
  \end{bmatrix}\end{align}

배치 작업(batching)은 더 많은 병렬 처리를 가능하게 하지만, 모델이 독립적으로 각 컬럼들을 취급해야 함을 뜻합니다;
예를 들어, 위 예제에서 ``G`` 와 ``F`` 의 의존성(dependance)은 학습되지 않습니다.

```python
from torchtext.datasets import WikiText2
from torchtext.data.utils import get_tokenizer
from torchtext.vocab import build_vocab_from_iterator

train_iter = WikiText2(split='train')
tokenizer = get_tokenizer('basic_english')
vocab = build_vocab_from_iterator(map(tokenizer, train_iter), specials=['<unk>'])
vocab.set_default_index(vocab['<unk>'])

def data_process(raw_text_iter: dataset.IterableDataset) -> Tensor:
    """Converts raw text into a flat Tensor."""
    data = [torch.tensor(vocab(tokenizer(item)), dtype=torch.long) for item in raw_text_iter]
    return torch.cat(tuple(filter(lambda t: t.numel() > 0, data)))

# train_iter was "consumed" by the process of building the vocab,
# so we have to create it again
train_iter, val_iter, test_iter = WikiText2()
train_data = data_process(train_iter)
val_data = data_process(val_iter)
test_data = data_process(test_iter)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def batchify(data: Tensor, bsz: int) -> Tensor:
    """Divides the data into bsz separate sequences, removing extra elements
    that wouldn't cleanly fit.

    Args:
        data: Tensor, shape [N]
        bsz: int, batch size

    Returns:
        Tensor of shape [N // bsz, bsz]
    """
    seq_len = data.size(0) // bsz
    data = data[:seq_len * bsz]
    data = data.view(bsz, seq_len).t().contiguous()
    return data.to(device)

batch_size = 20
eval_batch_size = 10
train_data = batchify(train_data, batch_size)  # shape [seq_len, batch_size]
val_data = batchify(val_data, eval_batch_size)
test_data = batchify(test_data, eval_batch_size)
```

입력(input) 과 타겟(target) 시퀀스를 생성하기 위한 함수들
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

``get_batch()`` 함수는 트랜스포머 모델을 위한 입력-타겟 시퀀스 쌍(pair)을 생성합니다.
이 함수는 소스 데이터를 ``bptt`` 길이를 가진 덩어리로 세분화 합니다.
언어 모델링 과제를 위해서, 모델은 다음 단어인 ``Target`` 이 필요 합니다.
예를 들어, ``bptt`` 의 값이 2 라면, 우리는 ``i`` = 0 일 때 다음의 2 개의 변수(Variable) 를 얻을 수 있습니다:

변수 덩어리는 트랜스포머 모델의 ``S`` 차원과 일치하는 0 차원에 해당합니다.
배치 차원 ``N`` 은 1 차원에 해당합니다.

```python
bptt = 35
def get_batch(source: Tensor, i: int) -> Tuple[Tensor, Tensor]:
    """
    Args:
        source: Tensor, shape [full_seq_len, batch_size]
        i: int

    Returns:
        tuple (data, target), where data has shape [seq_len, batch_size] and
        target has shape [seq_len * batch_size]
    """
    seq_len = min(bptt, len(source) - 1 - i)
    data = source[i:i+seq_len]
    target = source[i+1:i+1+seq_len].reshape(-1)
    return data, target
```

인스턴스(instance) 초기화하기
--------------------------------

모델의 하이퍼파라미터(hyperparameter)는 아래와 같이 정의됩니다.
단어 사이즈는 단어 오브젝트의 길이와 일치 합니다.

```python
ntokens = len(vocab) # 단어 사전(어휘집)의 크기
emsize = 200 # 임베딩 차원
d_hid = 200 # nn.TransformerEncoder 에서 피드포워드 네트워크(feedforward network) 모델의 차원
nlayers = 2 # nn.TransformerEncoder 내부의 nn.TransformerEncoderLayer 개수
nhead = 2 # nn.MultiheadAttention의 헤드 개수
dropout = 0.2 # 드랍아웃(dropout) 확률
model = TransformerModel(ntokens, emsize, nhead, d_hid, nlayers, dropout).to(device)
```

모델 실행하기
-------------

`CrossEntropyLoss <https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html>`__ 를
`SGD <https://pytorch.org/docs/stable/generated/torch.optim.SGD.html>`__ (확률적 경사 하강법) 옵티마이저(optimizer)와
함께 사용하였습니다. 학습률(learning rate)는 5.0으로 초기화하였으며 `StepLR <https://pytorch.org/docs/master/optim.html?highlight=steplr#torch.optim.lr_scheduler.StepLR>`__
스케쥴을 따릅니다. 학습하는 동안, `nn.utils.clip_grad_norm\_ <https://pytorch.org/docs/stable/generated/torch.nn.utils.clip_grad_norm_.html>`__
을 사용하여 기울기(gradient)가 폭발(exploding)하지 않도록 합니다.

```python
import copy
import time

criterion = nn.CrossEntropyLoss()
lr = 5.0  # 학습률(learning rate)
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)

def train(model: nn.Module) -> None:
    model.train()  # 학습 모드 시작
    total_loss = 0.
    log_interval = 200
    start_time = time.time()
    src_mask = generate_square_subsequent_mask(bptt).to(device)

    num_batches = len(train_data) // bptt
    for batch, i in enumerate(range(0, train_data.size(0) - 1, bptt)):
        data, targets = get_batch(train_data, i)
        batch_size = data.size(0)
        if batch_size != bptt:  # 마지막 배치에만 적용
            src_mask = src_mask[:batch_size, :batch_size]
        output = model(data, src_mask)
        loss = criterion(output.view(-1, ntokens), targets)

        optimizer.zero_grad()
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)
        optimizer.step()

        total_loss += loss.item()
        if batch % log_interval == 0 and batch > 0:
            lr = scheduler.get_last_lr()[0]
            ms_per_batch = (time.time() - start_time) * 1000 / log_interval
            cur_loss = total_loss / log_interval
            ppl = math.exp(cur_loss)
            print(f'| epoch {epoch:3d} | {batch:5d}/{num_batches:5d} batches | '
                  f'lr {lr:02.2f} | ms/batch {ms_per_batch:5.2f} | '
                  f'loss {cur_loss:5.2f} | ppl {ppl:8.2f}')
            total_loss = 0
            start_time = time.time()

def evaluate(model: nn.Module, eval_data: Tensor) -> float:
    model.eval()  # 평가 모드 시작
    total_loss = 0.
    src_mask = generate_square_subsequent_mask(bptt).to(device)
    with torch.no_grad():
        for i in range(0, eval_data.size(0) - 1, bptt):
            data, targets = get_batch(eval_data, i)
            batch_size = data.size(0)
            if batch_size != bptt:
                src_mask = src_mask[:batch_size, :batch_size]
            output = model(data, src_mask)
            output_flat = output.view(-1, ntokens)
            total_loss += batch_size * criterion(output_flat, targets).item()
    return total_loss / (len(eval_data) - 1)
```

에포크 내에서 반복됩니다. 만약 검증 오차(validation loss) 가 우리가 지금까지 관찰한 것 중 최적이라면 모델을 저장합니다.
매 에포크 이후에 학습률을 조절합니다.

```python
best_val_loss = float('inf')
epochs = 3
best_model = None

for epoch in range(1, epochs + 1):
    epoch_start_time = time.time()
    train(model)
    val_loss = evaluate(model, val_data)
    val_ppl = math.exp(val_loss)
    elapsed = time.time() - epoch_start_time
    print('-' * 89)
    print(f'| end of epoch {epoch:3d} | time: {elapsed:5.2f}s | '
          f'valid loss {val_loss:5.2f} | valid ppl {val_ppl:8.2f}')
    print('-' * 89)

    if val_loss < best_val_loss:
        best_val_loss = val_loss
        best_model = copy.deepcopy(model)

    scheduler.step()
```

평가 데이터셋(test dataset)으로 모델을 평가하기
-------------------------------------------------

```python
test_loss = evaluate(best_model, test_data)
test_ppl = math.exp(test_loss)
print('=' * 89)
print(f'| End of training | test loss {test_loss:5.2f} | '
      f'test ppl {test_ppl:8.2f}')
print('=' * 89)
```