# Learning disentangled representations in Flatland

This notebook uses our method to learn either entangled or disentangled representations in the Flatland environment (see Caselles-Dupré, Hugo, et al. "Flatland: a lightweight first-person 2-d environment for reinforcement learning." arXiv preprint arXiv:1809.00510 (2018).)

```python
import os
import gym
import math
import numpy as np
import time
from PIL import Image
import matplotlib.pyplot as plt
from IPython import display
import torch
import random
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

os.chdir('src/flatland/flat_game/')
from env import Env

```

    pygame 1.9.6
    Hello from the pygame community. https://www.pygame.org/contribute.html
    Loading chipmunk for Linux (64bit) [/home/william/.local/lib/python3.6/site-packages/pymunk/libchipmunk.so]

# Flatworld environment

We start by defining the flatworld environment, which is based on the code available at https://github.com/Caselles/NeurIPS19-SBDRL. This environment returns pixel observations of a ball on a cyclical 2D grid. The available (discrete) actions step the ball by a fixed amount in all four directions.

```python
RADIUS = 15
PERIOD = 10

class FlatWorld():

    class action_space():
        def __init__(self,n_actions):
            self.n = n_actions

        def sample(self, k=1):
            return torch.randint(0,self.n,(k,))

    class observation_space():
        def __init__(self):
            self.shape = [84,84]

    def __init__(self, env_parameters, period=10, radius=15):

        self.action_space = self.action_space(4)
        self.observation_space = self.observation_space()
        self.period = period

        self.step_size = 0.1*(63-2*env_parameters['agent']['radius'])/period
        start_positions_list = [27 + 10*self.step_size*i for i in range(period)]
        self.start_positions = []
        for i in start_positions_list:
            for j in start_positions_list:
                self.start_positions.append((i,j))

        env_parameters['agent']['radius'] = radius
        self.env = Env(**env_parameters)

    def reset(self, start_position=None):
        if start_position==None:
            obs = self.env.reset(position=random.sample(self.start_positions, 1)[0])
        else:
             obs = self.env.reset(position=start_position)
        return torch.FloatTensor(obs)/255

    def step(self, action):
        action_dict = self.create_action_dict(action)
        obs, reward, done, info = self.env.step(action_dict)
        return torch.FloatTensor(obs)/255

    def create_action_dict(self, action):
        action_dict = {}
        if action == 0:
            action_dict['longitudinal_velocity'] = 0
            action_dict['lateral_velocity'] = self.step_size
            action_dict['angular_velocity'] = 0
        if action == 1:
            action_dict['longitudinal_velocity'] = 0
            action_dict['lateral_velocity'] = -self.step_size
            action_dict['angular_velocity'] = 0
        if action == 2:
            action_dict['longitudinal_velocity'] = self.step_size
            action_dict['lateral_velocity'] = 0
            action_dict['angular_velocity'] = 0
        if action == 3:
            action_dict['longitudinal_velocity'] = -self.step_size
            action_dict['lateral_velocity'] = 0
            action_dict['angular_velocity'] = 0
        return action_dict

```

```python
agent_parameters = {
    'radius': 15,
    'speed': 10,
    'rotation_speed' : math.pi/8,
    'living_penalty': 0,
    'position': (30,30),
    'angle': 0,
    'sensors': [

        {
           'nameSensor' : 'proximity_test',
           'typeSensor': 'proximity',
           'fovResolution': 64,
           'fovRange': 300,
           'fovAngle': math.pi ,
           'bodyAnchor': 'body',
           'd_r': 0,
           'd_theta': 0,
           'd_relativeOrientation': 0,
           'display': False,
        }

    ],
    'actions': ['forward', 'turn_left', 'turn_right', 'left', 'right', 'backward'],
    'measurements': ['health', 'poisons', 'fruits'],
    'texture': {
        'type': 'color',
        'c': (255, 255, 255)
    },
    'normalize_measurements': False,
    'normalize_states': False,
    'normalize_rewards': False
}

env_parameters = {
    'map':False,
    'n_rooms': 2,
    'display': False,
    'horizon': 10001,
    'shape': (84, 84),
    'mode': 'time',
    'poisons': {
        'number': 0,
        'positions': 'random',
        'size': 10,
        'reward': -10,
        'respawn': True,
        'texture': {
            'type': 'color',
            'c': (255, 255, 255),
        }
    },
    'fruits': {
        'number': 0,
        'positions': 'random',
        'size': 10,
        'reward': 10,
        'respawn': True,
        'texture': {
            'type': 'color',
            'c': (255, 150, 0),
        }
    },
    'obstacles': [

    ],
    'walls_texture': {
        'type': 'color',
        'c': (1, 1, 1)
    },
    'agent': agent_parameters
}
```

**Now show a few consecutive states from this environment**

```python
import matplotlib.gridspec as gridspec

env = FlatWorld(env_parameters, period=PERIOD, radius=RADIUS)
plt.figure(figsize = (3,3))
gs1 = gridspec.GridSpec(3, 3)
gs1.update(wspace=0.02, hspace=0.02)
plt.grid(None)
state = env.reset()
for i in range(9):
    ax = plt.subplot(gs1[i])
    ax.axis('off')
    ax.set_aspect('equal')
    ax.imshow(state)
    display.display(plt.gcf())
    time.sleep(0.2)
    display.clear_output(wait=True)
    action = random.sample([0,1,2,3],k=1)[0]
    action = 2
    #print(env.env.agent.body.position)
    state = env.step(action)

plt.savefig("env.png", bbox_inches='tight')
```

### Latent space

**Encoder/Decoder**

Now we want to learn to represent this environment in some latent space (which we, for now, simply assume to be 4-dimensional).  We will require both an encoder and decoder, which will use convolutional neural networks.

```python
class Encoder(nn.Module):

    def __init__(self, n_out=4, n_hid = 64):

        super().__init__()

        self.conv = nn.Conv2d(1, 5, 10, stride=3)
        self.fc1 = nn.Linear(180, n_hid)
        self.fc2 = nn.Linear(n_hid, n_out)

    def forward(self, x):
        x = F.relu(self.conv(x.unsqueeze(0).unsqueeze(1)))
        x = F.max_pool2d(x, 4, 4)
        x = x.view(-1, 180)
        x = F.relu(self.fc1(x))
        return F.normalize(self.fc2(x)).squeeze()

class Decoder(nn.Module):

    def __init__(self, n_in=4, n_hid = 64):

        super().__init__()

        self.fc1 = nn.Linear(n_in, n_hid)
        self.fc2 = nn.Linear(n_hid, 180)
        self.conv = nn.ConvTranspose2d(5, 1, 34, stride=10)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = x.view(1,5,6,6)
        x = self.conv(x)
        return torch.sigmoid(x).squeeze()
```

```python
encoder = Encoder(n_out=4)
decoder = Decoder(n_in=4)
print(encoder)
print(decoder)
```

    Encoder(
      (conv): Conv2d(1, 5, kernel_size=(10, 10), stride=(3, 3))
      (fc1): Linear(in_features=180, out_features=64, bias=True)
      (fc2): Linear(in_features=64, out_features=4, bias=True)
    )
    Decoder(
      (fc1): Linear(in_features=4, out_features=64, bias=True)
      (fc2): Linear(in_features=64, out_features=180, bias=True)
      (conv): ConvTranspose2d(5, 1, kernel_size=(34, 34), stride=(10, 10))
    )

**Make sure dimensions match**

```python
obs = torch.FloatTensor(state)
print(obs.shape)
latent = encoder(obs)
print(latent.shape)
reconstructed = decoder(latent)
print(reconstructed.shape)
```

    torch.Size([84, 84])
    torch.Size([4])
    torch.Size([84, 84])

**Representation**

The crux of the matter is learning to 'represent' actions in the observation space with actions in latent space.  Here, we will do this by assuming every action is a generalized rotation in latent space, which we denote with a series of 2-dimensional rotations.

A 2-d rotation is given by:

\begin{pmatrix}
\cos(\theta) & \sin(\theta) \\
-\sin(\theta) & \cos(\theta)
\end{pmatrix}

and we denote a rotation in dimensions $i$ and $j$ of a higher dimensional space as $R_{i,j}(\theta)$.  For $i=1$, $j=4$, in a 4-dimensional space:

\begin{equation}
R_{1,4}(\theta) =
\begin{pmatrix}
\cos(\theta) & 0 & 0 & \sin(\theta) \\
0 & 1 & 0 & 0 \\
0 & 0 & 1 & 0 \\
-\sin(\theta) & 0 & 0 & \cos(\theta)
\end{pmatrix}
\end{equation}

An arbitrary rotation, denoted $g$ as I am subtly moving towards this being a group action, can then be written as:

\begin{equation}
    g(\theta_{1,2},\theta_{1,3},\dots,\theta_{n-1,n}) = \prod_{i=1}^{n-1} \prod_{j=1+1}^{n} R_{i,j}(\theta_{i,j})
\end{equation}

which has $n(n-1)/2$ free parameters (i.e. $\theta_{i,j}$'s).

```python
class Representation():

    def __init__(self, dim=4):
        self.dim = dim
        self.params = dim*(dim-1)//2
        self.thetas = torch.autograd.Variable(np.pi*(2*torch.rand(self.params)-1)/dim, requires_grad=True)

        self.__matrix = None

    def set_thetas(self, thetas):
        self.thetas = thetas
        self.thetas.requires_grad = True
        self.clear_matrix()

    def clear_matrix(self):
        self.__matrix = None

    def get_matrix(self):
        if self.__matrix is None:
            k = 0
            mats = []
            for i in range(self.dim-1):
                for j in range(self.dim-1-i):
                    theta_ij = self.thetas[k]
                    k+=1
                    c, s = torch.cos(theta_ij), torch.sin(theta_ij)

                    rotation_i = torch.eye(self.dim, self.dim)
                    rotation_i[i, i] = c
                    rotation_i[i, i+j+1] = s
                    rotation_i[j+i+1, i] = -s
                    rotation_i[j+i+1, j+i+1] = c

                    mats.append(rotation_i)

            def chain_mult(l):
                if len(l)>=3:
                    return l[0]@l[1]@chain_mult(l[2:])
                elif len(l)==2:
                    return l[0]@l[1]
                else:
                    return l[0]

            self.__matrix = chain_mult(mats)

        return self.__matrix
```

**LatentWorld**

Now, for symmetry's sake, we'll also have a `LatentWorld` which acts as the environment in the latent space.

```python
class LatentWorld():

    class action_space():
        def __init__(self,n_actions):
            self.n = n_actions

        def sample(self, k=1):
            return torch.randint(0,self.n,(k,))

    class observation_space():
        def __init__(self,n_features):
            self.shape = [n_features]

    def __init__(self,
                 dim=4,
                 n_actions=4,
                 action_reps=None):

        self.dim = dim

        self.action_space = self.action_space(n_actions)
        self.observation_space = self.observation_space(dim)

        if action_reps is None:
            self.action_reps = [Representation(dim=self.dim) for _ in range(n_actions)]
        else:
            if len(action_reps)!=n_actions:
                raise Exception("Must pass an action representation for every action.")
            if not all([rep.dim==self.dim]):
                raise Exception("Action representations do not act on the dimension of the latent space.")
            self.action_reps = action_reps

    def reset(self, state_init):
        self.state = state_init
        return self.get_observation()

    def clear_representations(self):
        for rep in self.action_reps:
            rep.clear_matrix()

    def get_representation_params(self):
        params = []
        for rep in self.action_reps:
            params.append(rep.thetas)
        return params

    def save_representations(self, path):
        if os.path.splitext(path)[-1] != '.pth':
            path += '.pth'
        rep_thetas = [rep.thetas for rep in self.action_reps]
        return torch.save(rep_thetas, path)

    def load_reprentations(self, path):
        rep_thetas = torch.load(path)
        for rep in self.action_reps:
            rep.set_thetas(rep_thetas.pop(0))

    def get_observation(self):
        return self.state

    def step(self,action):
        self.state = torch.mv(self.action_reps[action].get_matrix(), self.state)
        obs = self.get_observation()
        return obs
```

## 3. Training

So the basic training loop is pretty straightfoward.  We simply play out episodes from random starting configurations, encoded by the `Encoder`, for `ep_steps` time-steps.  Each random action is executed in both the `FlatWorld` and the `LatentWorld`, and then the latent state is transformed to the observation space by the `Decoder` where the loss function measures its deviation from the true state.

Note: sometimes training without the disentanglement regularization fails to find toroidal structure, especially when the radius of the ball is very small.

```python
dim = 4

obs_env = FlatWorld(env_parameters, period=PERIOD, radius=RADIUS)
lat_env = LatentWorld(dim = dim,
                      n_actions = obs_env.action_space.n)
decoder = Decoder(n_in = dim, n_hid = 64)
encoder = Encoder(n_out = dim, n_hid = 64)

optimizer_dec = optim.Adam(decoder.parameters(),
                           lr=1e-2,
                           weight_decay=0)

optimizer_enc = optim.Adam(encoder.parameters(),
                           lr=1e-2,
                           weight_decay=0)

optimizer_rep = optim.Adam(lat_env.get_representation_params(),
                           lr=1e-2,
                           weight_decay=0)

losses = []
```

```python
n_sgd_steps = 3000
ep_steps = 5
batch_eps = 16

i = 0

t_start = time.time()

temp = 0

while i < n_sgd_steps:

    loss = torch.zeros(1)

    for _ in range(batch_eps):
        t_ep = -1
        while t_ep < ep_steps:
            if t_ep == -1:
                obs_x = obs_env.reset()
                obs_z = lat_env.reset(encoder(obs_x))
            else:
                action = obs_env.action_space.sample().item()
                obs_x = obs_env.step(action)
                obs_z = lat_env.step(action)

            t_ep += 1

            obs_x_recon = decoder(obs_z)

            loss += F.binary_cross_entropy(obs_x_recon, obs_x)

    loss /= (ep_steps*batch_eps)

    losses.append(loss.item())

    optimizer_dec.zero_grad()
    optimizer_enc.zero_grad()
    optimizer_rep.zero_grad()
    loss.backward()
    optimizer_enc.step()
    optimizer_dec.step()
    optimizer_rep.step()

    # Remember to clear the cached action representations after we update the parameters!
    lat_env.clear_representations()

    i+=1

    if i%10==0:
        print("iter {} : loss={:.3f} : last 10 iters in {:.3f}s".format(i, loss.item(), time.time() - t_start),
              end="\r" if i%100 else "\n")
        t_start = time.time()
```

## 4. Testing

Testing is easy too, we just play out an episode and see how well the reconstructed image agrees with the ground truth!

```python
def plot_state(obs, ax):
    ax.imshow(obs)
    ax.set_aspect('equal')
    ax.set_xticks([])
    ax.set_yticks([])

    return ax

n_steps = 10

fig, (ax1,ax2) = plt.subplots(1, 2)

ax1.set_title("Ground truth")
ax2.set_title("Reconstruction")

for i in range(n_steps+1):

    if i==0:
        action = "N\A"
        obs_x = obs_env.reset()
        obs_z = lat_env.reset(encoder(obs_x))
    else:
        action = obs_env.action_space.sample().item()
        obs_x = obs_env.step(action)
        obs_z = lat_env.step(action)

    obs_x_recon = decoder(obs_z)

    fig.suptitle('step {} : last action = {}'.format(i, action), fontsize=16)

    plot_state(obs_x.detach().numpy(),ax1)
    plot_state(obs_x_recon.detach().numpy(),ax2)

    display.clear_output(wait=True)
    display.display(plt.gcf())
    time.sleep(0.5)

display.clear_output(wait=False)
```

We will now have a look at the latent space, we will make a 2D projection of the 4D latent space for every possible frame (There are 121 possible frames in this environment). Note that since we use random projections, in some cases the toric structure we find is more obvious than in others.

**Positions in latent space**

```python
from sklearn.random_projection import GaussianRandomProjection

latent_points = []

for start_position in obs_env.start_positions:
    obs = obs_env.reset(start_position=start_position)
    latent = encoder(obs)
    latent_points.append(latent.detach().tolist())

latent_map = np.array(latent_points)
```

```python
period = obs_env.period

color=['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf',"#2fa36b"]
marks=["1","2","3","4","+",">","<","^","v","x","d"]
pca = GaussianRandomProjection(n_components=2)

latent_2d = pca.fit_transform(latent_map)

fig = plt.figure(figsize=(6,4))
ax = fig.add_subplot(111)#, projection='3d')
s=[120]*5+[50]*6
for i in range (period**2):
    ax.scatter(x=latent_2d.transpose()[0][i],
                y=latent_2d.transpose()[1][i],
                c=color[i//period],
                s=s[i%period],
                marker=marks[i%period])
plt.title('Representations - Our method',fontsize=16)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
fig.show()
plt.savefig("latent_flatland.png", bbox_inches='tight')
```

**2) Plot Action Representations**

```python
width=0.5

rep_thetas = [rep.thetas.detach().numpy() for rep in lat_env.action_reps]

for rep in lat_env.action_reps:
    print(rep.get_matrix())
    print(torch.matrix_power(rep.get_matrix(), 5))

plt_lim = max( 0.12, max([max(t) for t in rep_thetas])/(2*np.pi) )
titles = ["up", "down", "right", "left"]

with plt.style.context('seaborn-paper', after_reset=True):

    fig, axs = plt.subplots(1, len(rep_thetas), figsize=(15, 3), gridspec_kw={"wspace":0.4})

    for i, thetas in enumerate(rep_thetas):
        x = np.arange(len(thetas))
        axs[i].bar(x - width/2, thetas/(2*np.pi), width, label='Rep {}'.format(i))
        axs[i].hlines((0.2,-0.2), -2., 7., linestyles="dashed")
        axs[i].hlines(0., -2., 7.)
        axs[i].set_xticks(x-0.25)
        axs[i].set_xticklabels(["12","13","14","23","24","34"], fontsize = 15)
        axs[i].set_xlabel("$ij$", fontsize = 15)

        axs[i].set_ylim(-plt_lim,plt_lim)
        axs[i].set_xlim(-.75, 5.75)
        axs[i].set_title(titles[i], fontsize = 15)

        axs[i].tick_params(labelsize=15)

    axs[0].set_ylabel(r"$\theta / 2\pi$", fontsize = 15)
    plt.savefig("action_rep_entangled.png", bbox_inches='tight')

```

## 5. Disentanglement

***Some jargon***

It's nice that it works, but the real point here is to try and learn a *disentangled* representation of the actions.

Before considering how best to do this, we want to define a metric of 'disentanglement'.  We consider the evolution of an observable (latent) vector, $x \in X$ ($z \in Z$), under the element $g \in G$ of the group of symmetries generating transformations of the object.  Then we are looking for a representation, $\rho:G \rightarrow GL(V)$, such that the transformation is linear in the latent space, i.e.
\begin{equation}
    z^{\prime} = \rho(g) \cdot z.
\end{equation}
Note, in our case, the representations are the rotation matrices we learn.

For this representation to be disentangled, it means that if there exists a subgroup decomposition of $G$
\begin{equation}
    G = G_1 \times G_2 \times \dots \times G_n,
\end{equation}
then we equivalently decompose the representation, $(\rho, G)$, into subrepresentations:
\begin{equation}
    V = V_1 \oplus V_2 \oplus \dots \oplus V_n
\end{equation}
such that the restricted subrepresentations $(\rho_{\vert G_i}, V_i)_i$ are non-trivial, and the restricted subrepresentations $(\rho_{\vert G_i}, V_j)_{j \neq i}$ are trivial.

In our context, a GridWorld with 5 points in each dimension is represented by $G = C_5 \times C_5$ (where $C_5$ is the cyclic group).  This is a subgroup of $\mathrm{SO}(2) \times \mathrm{SO}(2)$, therefore we hope to find the disentangled representation of the actions (up, down, left, right) that corresponds to this.

***Some practicalities***

Our intuition is that the disentangled representation acts as the identity on as many dimensions as possible.  We could attempt to enforce this with some regularization during training.  Normal weight decay won't cut it, as that tries to reduce all weights, where as what we really want to do is have all *but one* of our thetas (which corresponds to the rotation/coupling of two dimensions) to be zero.

**1. Entanglement regularisation**

So for $m$ parameters, ${\theta_1, \dots, \theta_m}$, we want to regularise with
\begin{equation}
    \sum_{i \neq j} \vert\theta_i\vert^2, \mathrm{where\ } \theta_j {=} \mathrm{max_k}({\vert\theta_k\vert}).
\end{equation}
We will also use this term as our metric of 'entanglement'.

```python
def calc_entanglement(params):
    params = params.abs().pow(2)
    return params.sum() - params.max()

params = torch.FloatTensor([1,1,0.5,0,0])
calc_entanglement(params)
```

    tensor(1.2500)

### Training with regularization

```python
dim = 4

obs_env = FlatWorld(env_parameters, period=PERIOD, radius=RADIUS)
lat_env = LatentWorld(dim = dim,
                      n_actions = obs_env.action_space.n)
decoder = Decoder(n_in = dim, n_hid = 64)
encoder = Encoder(n_out = dim, n_hid = 64)

optimizer_dec = optim.Adam(decoder.parameters(),
                           lr=1e-2,
                           weight_decay=0)

optimizer_enc = optim.Adam(encoder.parameters(),
                           lr=1e-2,
                           weight_decay=0)

optimizer_rep = optim.Adam(lat_env.get_representation_params(),
                           lr=1e-2,
                           weight_decay=0)

losses = []
entanglement = []
```

```python
n_sgd_steps = 3000
ep_steps = 5
batch_eps = 16
entanglement_target = 0

i = 0

t_start = time.time()

temp = 0

while i < n_sgd_steps:

    loss = torch.zeros(1)

    for _ in range(batch_eps):
        t_ep = -1
        while t_ep < ep_steps:
            if t_ep == -1:
                obs_x = obs_env.reset()
                obs_z = lat_env.reset(encoder(obs_x))
            else:
                action = obs_env.action_space.sample().item()
                obs_x = obs_env.step(action)
                obs_z = lat_env.step(action)

            t_ep += 1

            obs_x_recon = decoder(obs_z)

            loss += F.binary_cross_entropy(obs_x_recon, obs_x)

    loss /= (ep_steps * batch_eps)
    raw_loss = loss.item()

    reg_loss = sum([calc_entanglement(r.thetas) for r in lat_env.action_reps])/4

    loss += (reg_loss-entanglement_target).abs() * 1e-2

    losses.append(raw_loss)
    entanglement.append(reg_loss.item())

    optimizer_dec.zero_grad()
    optimizer_enc.zero_grad()
    optimizer_rep.zero_grad()
    loss.backward()
    optimizer_enc.step()
    optimizer_dec.step()
    optimizer_rep.step()

    # Remember to clear the cached action representations after we update the parameters!
    lat_env.clear_representations()

    i+=1

    if i%10==0:
        print("iter {} : loss={:.3f} : entanglement={:.2e} : last 10 iters in {:.3f}s".format(
            i, raw_loss, reg_loss.item(), time.time() - t_start
            ), end="\r" if i%100 else "\n")
        t_start = time.time()
```

    iter 100 : loss=0.196 : entanglement=1.28e-01 : last 10 iters in 2.261s
    iter 200 : loss=0.110 : entanglement=1.64e-01 : last 10 iters in 2.717s
    iter 300 : loss=0.079 : entanglement=1.37e-01 : last 10 iters in 2.428s
    iter 400 : loss=0.068 : entanglement=1.21e-01 : last 10 iters in 2.503s
    iter 500 : loss=0.058 : entanglement=9.97e-02 : last 10 iters in 2.390s
    iter 600 : loss=0.052 : entanglement=7.73e-02 : last 10 iters in 2.173s
    iter 700 : loss=0.047 : entanglement=6.18e-02 : last 10 iters in 2.457s
    iter 720 : loss=0.047 : entanglement=5.78e-02 : last 10 iters in 2.508s

    /home/william/Bureau/Python/gantime/rep-learning/paris/src/flatland/flat_game/sensors/proximity_sensor.py:42: RuntimeWarning: invalid value encountered in not_equal
      mask = resized_img != 0

    iter 800 : loss=0.045 : entanglement=4.67e-02 : last 10 iters in 2.678s
    iter 900 : loss=0.045 : entanglement=3.69e-02 : last 10 iters in 3.733s
    iter 1000 : loss=0.041 : entanglement=2.65e-02 : last 10 iters in 2.215s
    iter 1100 : loss=0.042 : entanglement=2.10e-02 : last 10 iters in 2.264s
    iter 1200 : loss=0.040 : entanglement=1.57e-02 : last 10 iters in 2.423s
    iter 1300 : loss=0.040 : entanglement=1.22e-02 : last 10 iters in 2.254s
    iter 1400 : loss=0.039 : entanglement=8.43e-03 : last 10 iters in 3.703s
    iter 1500 : loss=0.040 : entanglement=7.17e-03 : last 10 iters in 3.049s
    iter 1600 : loss=0.039 : entanglement=5.31e-03 : last 10 iters in 2.675s
    iter 1700 : loss=0.037 : entanglement=4.04e-03 : last 10 iters in 2.599s
    iter 1800 : loss=0.039 : entanglement=3.47e-03 : last 10 iters in 2.330s
    iter 1900 : loss=0.038 : entanglement=2.71e-03 : last 10 iters in 2.874s
    iter 2000 : loss=0.038 : entanglement=2.13e-03 : last 10 iters in 2.481s
    iter 2100 : loss=0.042 : entanglement=1.38e-03 : last 10 iters in 2.716s
    iter 2200 : loss=0.037 : entanglement=1.36e-03 : last 10 iters in 2.523s
    iter 2300 : loss=0.036 : entanglement=9.73e-04 : last 10 iters in 3.021s
    iter 2400 : loss=0.036 : entanglement=8.62e-04 : last 10 iters in 2.309s
    iter 2500 : loss=0.036 : entanglement=5.97e-04 : last 10 iters in 2.594s
    iter 2600 : loss=0.037 : entanglement=5.08e-04 : last 10 iters in 2.333s
    iter 2700 : loss=0.036 : entanglement=5.65e-04 : last 10 iters in 2.319s
    iter 2800 : loss=0.037 : entanglement=6.81e-04 : last 10 iters in 2.877s
    iter 2900 : loss=0.037 : entanglement=2.65e-04 : last 10 iters in 2.333s
    iter 3000 : loss=0.035 : entanglement=1.87e-04 : last 10 iters in 2.304s

### Testing: action representations

```python
width=0.5

rep_thetas = [rep.thetas.detach().numpy() for rep in lat_env.action_reps]

for rep in lat_env.action_reps:
    print(rep.get_matrix())
    print(torch.matrix_power(rep.get_matrix(), 5))

plt_lim = max( 0.12, max([max(t) for t in rep_thetas])/(2*np.pi) )
titles = ["up", "down", "right", "left"]

with plt.style.context('seaborn-paper', after_reset=True):

    fig, axs = plt.subplots(1, len(rep_thetas), figsize=(20, 3), gridspec_kw={"wspace":0.4})

    for i, thetas in enumerate(rep_thetas):
        x = np.arange(len(thetas))
        axs[i].bar(x - width/2, thetas/(2*np.pi), width, label='Rep {}'.format(i))
        axs[i].hlines((0.2,-0.2), -2., 6., linestyles="dashed")
        axs[i].hlines(0., -2., 6.)
        axs[i].set_xticks(x-0.25)
        axs[i].set_xticklabels(["12","13","14","23","24","34"], fontsize = 15)
        axs[i].set_xlabel("$ij$", fontsize = 15)

        axs[i].set_ylim(-plt_lim,plt_lim)
        axs[i].set_xlim(-.75, 5.5)
        axs[i].set_title(titles[i], fontsize = 15)

        axs[i].tick_params(labelsize=15)

    axs[0].set_ylabel(r"$\theta / 2\pi$", fontsize = 15)
    plt.savefig("action_rep_entangled.png", bbox_inches='tight')

```

**Show predictions made by trained network with disentangled representations**

```python
def plot_state(obs, ax):
    ax.imshow(obs)
    ax.set_aspect('equal')
    ax.set_xticks([])
    ax.set_yticks([])

    return ax

n_steps = 10

fig, (ax1,ax2) = plt.subplots(1, 2)

ax1.set_title("Ground truth")
ax2.set_title("Reconstruction")

for i in range(n_steps+1):

    if i==0:
        action = "N\A"
        obs_x = obs_env.reset()
        obs_z = lat_env.reset(encoder(obs_x))
    else:
        action = obs_env.action_space.sample().item()
        obs_x = obs_env.step(action)
        obs_z = lat_env.step(action)

    obs_x_recon = decoder(obs_z)

    fig.suptitle('step {} : last action = {}'.format(i, action), fontsize=16)

    plot_state(obs_x.detach().numpy(),ax1)
    plot_state(obs_x_recon.detach().numpy(),ax2)

    display.clear_output(wait=True)
    display.display(plt.gcf())
    time.sleep(0.5)

display.clear_output(wait=False)
```

**Show 2D projections of learned representations in latent space**

```python
from sklearn.random_projection import GaussianRandomProjection

latent_points = []

for start_position in obs_env.start_positions:
    obs = obs_env.reset(start_position=start_position)
    latent = encoder(obs)
    latent_points.append(latent.detach().tolist())

latent_map = np.array(latent_points)
```

```python
period = obs_env.period

color=['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf',"#2fa36b",'#2f14fb','#ada854','#c353cc','#1c1392','#8eeeb0']
marks=["1","2","3","4","+",">","<","^","v","x","d","p","P","X",'_','|']
pca = GaussianRandomProjection(n_components=2)

latent_2d = pca.fit_transform(latent_map)

fig = plt.figure(figsize=(6,4))
ax = fig.add_subplot(111)#, projection='3d')
s=[120]*5+[50]*6
for i in range (period**2):
    ax.scatter(x=latent_2d.transpose()[0][i],
                y=latent_2d.transpose()[1][i],
             #   zs=latent_2d.transpose()[2][i],
                c=color[i//period],
                #s=s[i%period],
                marker=marks[i%period])
 #   ax.set_xlim(-.6/1.4,.6/1.4)
  #  ax.set_ylim(-.8/1.4,.8/1.4)
  #  ax.set_zlim(-1./1.6,1./1.6)
    #ax.view_init(elev=45, azim=45)
plt.title('Representations - Our method',fontsize=16)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
fig.show()
plt.savefig("latent_flatland.png", bbox_inches='tight')
```


```python
%matplotlib inline

import matplotlib
import numpy as np
import matplotlib.pyplot as plt

plt.rcParams["figure.figsize"] = (12, 9)
plt.rcParams["font.size"] = 18
```

# Reactor Kinetics

Reactor kinetics focuses on the relationship between criticality, power, and time. Delayed neutrons and reactor control are at the heart of reactor kinetics. As changing conditions in the core impact neutron multiplication, control rods, burnable poisons, and chemical shim may be introduced to keep $k_{eff}$ near 1.

Time-dependent fluctuations in neutron population, fluid flow, and heat transfer are essential to understanding the performance and safety of a reactor. Such transients include normal reactor startup and shutdown as well as abnormal scenarios including Beyond Design Basis Events (BDBEs) such as Accident Transients Without Scram (ATWS).

## Learning Objectives

At the end of this lesson, you will be equipped to:

- State the relationship between criticality and reactivity
- State the Point Reactor Kinetics Equations
- Describe temperature feedback of reactivity
- Apply the Point Reactor Kinetics Equations

## Fission and Delayed Neutrons

<large>$$\sigma(E,\vec{r},\hat{\Omega},T,t,x,i)$$</large>

$k_{eff}$

i

                                          $\beta_i$

 \begin{align}
		     k &= \mbox{"neutron multiplication factor"}\\
		     &= \frac{\mbox{neutrons causing fission}}{\mbox{neutrons produced by fission}}\\
		     \rho &= \frac{k-1}{k}\\
		     \rho &= \mbox{reactivity}\\
\end{align}

\begin{align}
\rho(t) = \rho_0 + \rho_f(t) + \rho_{ext}
\end{align}

where

\begin{align}
   \rho(t) &= \mbox{total reactivity}\\
   \rho_f(t) &= \mbox{reactivity from feedback}\\
   \rho_{ext}(t) &= \mbox{external reactivity insertion}\\
   \rho_f(t) &= \sum_i \alpha_i\delta T_i\\
   T_i &= \mbox{temperature of component i}\\
   \alpha_i &= \mbox{temperature reactivity coefficient of i}.
\end{align}

## Point Reactor Kinetics Equations

Simplifying assumptions:

- The reactor is a point
- Assume all delayed neutron precursors have the same decay constant, $\lambda$.
- Let C(t) be the total number of delayed neutron precursors at time t.

In each neutron cycle,  $k_{eff}n(t)$ is the number of new neutrons eventually produced, and is a combination of both prompt and delayed neutrons.

\begin{align}
(1-\beta)k_{eff}n(t) &= \mbox{ prompt neutrons at the end of cycle}\\
\beta k_{eff}n(t) &= \mbox{ delayed neutron precursors produced in the cycle}\\
\mathscr{l}' &= \mbox{ cycle length of each cycle}\\
\frac{\beta k_{eff}n(t)}{\mathscr{l}'} &= \mbox{ delayed neutron precursors per unit time}\\
\lambda C(t) &= \mbox{ rate of decay by precursors}
\end{align}

Thus, the net rate of increase in the number of precursors is :

\begin{align}
\frac{dC(t)}{dt} &= \frac{\beta}{\mathscr{l}'}n(t) - \lambda C(t)\\
\end{align}

### In each cycle:

\begin{align}
n(t) &= \mbox{ neutrons disappear}\\
(1-\beta)k_{eff}n(t) &= \mbox{ prompt neutrons are produced by fission}\\
\frac{\left[(1-\beta)k_{eff} - 1\right]n(t)}{\mathscr{l}'} &=\mbox{ net rate of increase by prompt neutrons}\\
\lambda C(t) &= \mbox{ rate of neutron production by delayed neutron precursors}\\
S(t) &= \mbox{ rate of neutron production by non-fission sources}
\end{align}

Thus, the rate of increase in neutron population is the sum of production mechanisms:

\begin{align}
\frac{dn(t)}{dt} &= \frac{1}{\mathscr{l}'}\left[(1-\beta)k_{eff} - 1\right]n(t) + \lambda C(t) + S(t)\\
 &= \frac{k_{eff}}{\mathscr{l}'}\left[\frac{k_{eff} -1}{k_{eff}} - \beta \right]n(t) + \lambda C(t) + S(t)\\
\end{align}

We can define the effective neutron lifetime as $\Lambda = \mathscr{l} = \frac{\mathscr{l}'}{k_{eff}}$ to simplify this equation:

\begin{align}
\frac{dn(t)}{dt} &= \frac{\rho - \beta}{\mathscr{l}}n(t) + \lambda C(t) + S(t)\\
\frac{dC(t)}{dt} &= \frac{\beta}{\mathscr{l}}n(t) - \lambda C(t) \\
\end{align}

This can be solved by assuming a solution of the form:

\begin{align}
n(t) = \phi(t) = Ae^{\omega t}\\
C(t) = C_0 e^{\omega t}
\end{align}

### Multiple Delayed Neutron Precursor Groups

In reality, the delayed neutron precursors have very different halflives. As there are dozens of delayed neutron precursors, a typical strategy is to group these precurors into a small number of groups with similar halflives, as in the table below.

|Group	| Half-Life (s)	| Decay Constant (s−1)	| Energy (keV) |	Yield	| Fraction |
|-------------|-------------|-------------|-------------|-------------|-------------|
|1 |	55.72 |	0.0124 |	250 |	0.00052 |	0.000215 |
|2 |	22.72 |	0.0305 |	560 |	0.00346 |	0.001424 |
|3 |	6.22 |	0.111 |	405 |	0.00310 |	0.001274 |
|4 |	2.30 |	0.301 |	450 |	0.00624 |	0.002568 |
|5 |	0.610 |	1.14 |	- |	0.00182 |	0.000748 |
|6 |	0.230 |	3.01 |	- |	0.00066 |	0.000273 |

\begin{align}
\frac{dn(t)}{dt} &= \frac{\rho - \beta}{\mathscr{l}}n(t) + \sum_{i=1}^G\lambda_iC_i(t) + S(t)\\
\frac{dC_i(t)}{dt} &= \frac{\beta_i}{\mathscr{l}}n(t) - \lambda_iC_i(t)\\
\beta &= \sum_i^G \beta_i\\
i&= 1,...,G\\
\end{align}

## Feedback

Putting all of this together, the point reactor kinetics equations, with feedback, result in a "stiff" set of PDEs:

\begin{align}
\frac{d}{dt}\left[
    \begin{array}{c}
      p\\
      \zeta_1\\
      .\\
      \zeta_j\\
      .\\
      \zeta_J\\
      \omega_1\\
      .\\
      \omega_k\\
      .\\
      \omega_K\\
      T_{i}\\
      .\\
      T_{I}\\
    \end{array}
    \right]
    =
    \left[
      \begin{array}{ c }
        \frac{\rho(t,T_{i},\cdots)-\beta}{\Lambda}p +
        \displaystyle\sum^{j=J}_{j=1}\lambda_{d,j}\zeta_j\\
        \frac{\beta_1}{\Lambda} p - \lambda_{d,1}\zeta_1\\
        .\\
        \frac{\beta_j}{\Lambda}p-\lambda_{d,j}\zeta_j\\
        .\\
        \frac{\beta_J}{\Lambda}p-\lambda_{d,J}\zeta_J\\
        \kappa_1p - \lambda_{FP,1}\omega_1\\
        .\\
        \kappa_kp - \lambda_{FP,k}\omega_k\\
        .\\
        \kappa_{k p} - \lambda_{FP,k}\omega_{k}\\
        f_{i}(p, C_{p,i}, T_{i}, \cdots)\\
        .\\
        f_{I}(p, C_{p,I}, T_{I}, \cdots)\\
      \end{array}
      \right]
      \end{align}
\begin{align}
    p &= \mbox{ reactor power }\\
    \rho(t,&T_{fuel},T_{cool},T_{mod}, T_{refl}) = \mbox{ reactivity}\\
    \beta &= \mbox{ fraction of neutrons that are delayed}\\
    \beta_j &= \mbox{ fraction of delayed neutrons from precursor group j}\\
    \zeta_j &= \mbox{ concentration of precursors of group j}\\
    \lambda_{d,j} &= \mbox{ decay constant of precursor group j}\\
    \Lambda &= \mbox{ mean generation time }\\
    \omega_k &= \mbox{ decay heat from FP group k}\\
    \kappa_k &= \mbox{ heat per fission for decay FP group k}\\
    \lambda_{FP,k} &= \mbox{ decay constant for decay FP group k}\\
    T_i &= \mbox{ temperature of component i}
\end{align}

## Units of Reactivity

### Delayed neutron fraction

In all of this, recall that the **delayed neutron fraction** is thus:

\begin{align}
\beta &= \mbox{Delayed neutron fraction}\\
&=\frac{\mbox{precursor atoms}}{\mbox{prompt neutrons }+\mbox{ precursor atoms}}\\
&= \frac{\mbox{delayed neutrons}}{\mbox{prompt neutrons }+\mbox{ delayed neutrons}}
\end{align}

The delayed neutron fraction **$\beta$ varies by fission isotope.**
- For a reactor where all fissions are from $^{235}U$, this fraction is $\beta_{235U} = 0.0064$.
- In $^{238}U$, the fraction is lower, $\beta_{238U} = 0.00157$.
- And $^{239}Pu$ generates even fewer delayed neutrons per fission $\beta_{239Pu} = 0.002$.

#### Think-pair-share
**Can you think of a physical reason that $\beta$ should vary by fission isotope?**

The **effective delayed neutron fraction** ($\beta_{eff}$) varies dramatically by reactor because:

- in most reactors, not all fissions are from $^{235}U$
- breeding and burning occurs over time, $\beta$ changes with burnup

### Dollar

We define one dollar of reactivity in a particular reactor as $\frac{\rho}{\beta}$.

### Cent

A cent is $\frac{1}{100^{th}}$ of a dollar of reactivity, so it's $\frac{\rho}{100\beta}$.

### Per Cent Mille (pcm)

A cent is  $\frac{1}{100^{th}}$ and one mille is $\frac{1}{1000^{th}}$ of a dollar.
Thus, one per cent mille (pcm) of a dollar is $10^{-5}\frac{\rho}{\beta}$.

## The Delayed Neutron Fraction and Criticality

### Subcriticality
$k<1$ is a subcritical reactor. This is immediate and all subcriticality is effectively prompt, though delayed neutrons have a slight impact.

### Supercriticality
$k>1$ is a supercritical reactor.

### Prompt
If the reactor is supercritical on prompt neutrons alone, then it is *prompt supercritical*. This is **bad** because control rods take more than $10^{-14}s$ to drop. Prompt supercriticality occurs when reactivity is equal to one dollar :

\begin{align}
\rho \ge \beta_{eff}
\end{align}

### Delayed

If the reactor is supercritical only if delayed neutrons are included, then it is just supercritical, or *delayed supercritical*. Delayed, controllable supercriticality occurs when reactivity is positive but below one dollar :

\begin{align}
0 < \rho < \beta_{eff}
\end{align}

## Example

Let's imagine a reactor with $\beta_{eff} = 0.006$
\begin{align}
k_{eff} = 0.99 \\
\rho &= \frac{k_{eff} - 1}{k_{eff}} \\
&= -0.01
\end{align}

This reactivity, in units of dollars, for this reactor, is:
\begin{align}
\frac{\rho}{\beta} &= \frac{-0.01}{0.006}\\
&= -1.67 [$]\\
&= -167 [cents]\\
\end{align}

The same reactivity in a reactor with  $\beta_{eff} = 0.005$:

\begin{align}
\frac{\rho}{\beta} &= \frac{-0.01}{0.005}\\
&= -2.00 [$]\\
&= -200 [cents]\\
\end{align}

```python

```


# Extended Rant Version

> This version needs significant refactoring. It is fairly complete tough.

## What happened

I do not normally write blog posts.

Today is not a normal day tough.

I believe that I have found a missing link in my understanding of bayesian modelling theory.

This is not so much about the bayes belief update equation. It's good. The problem is that the notation is confusing, making it difficult to distinguish and source(compute, obtain) the various terms in the equation, in connection to the real-world phenomena. And then, when solving a real world problem, this is not even where you put most of the time-effort in.

You might think that bayes update rule:

$p( A | B) = \frac{p(B|A)p(A)}{p(B)}$

is all that it takes.

This form is nice for proving that the bayes' theorem is correct.

However, what do these terms mean? How does one apply this?

You might have heard that $A$ is a model parameter, and $B$ is observation. And that you can derive the model parameter $A$ from observations.

Still, it is not at all obvious to someone that tries to figure it out for himself. Sources of confusion include:

* `probability` and `belief` are not the same thing, yet they are denoted with the same letter
* $p(B|A)$ does not need to sum to 1.0 while $p(A|B)$ does ?
* values of $p(B|A)$ need to be between 0 and 1, while values of $p(A|B)$ do not ?
* The name and function body for $p(A)$ and $p(B)$ is **completely** different, even tough they use the same symbols?
* Some symbols evaluate to scalars, and others evaluate to vector? then, some places that used to be scalar can be vectors sometimes?
* Data (observations) seems to have the same type as probability (`float`) , even tough the data can be `nominal`, like `T`/`H`, distribution, or vector what then?
* `Discrete events`, `datapoints`, `sets` and `distributions` are very different things, yet they are denoted by the same symbol and *location*
* `latent model parameters` versus `observable data` -- and where they come from, and where they go into? these are conceptually very different, yet they are denoted by location in a composite symbol as if they were interchangeable. Granted, the interchangeability is proven and even exploited in deriving the Bayes update rule: $p(A|B) p(B) = p(B|A)p(A)$ for any meaning of $A$ and $B$. However, this neat algebraic trick does no favours to the conceptual understanding and practical applications of this equation.
* Most books go straight to continuous distributions and multidimensional variables. These are beautiful exhibits of maths' notations' brevity and generalism, but also are absolutely redundant. Discrete events are fine. Scalars are fine. The additional hoop created by going straight to things like fractional dimensions is a serious barrier for the children AND their teachers.
* `class of function`, `instance of function`, `function name`, `function body` and `value of function` are all separate concepts in computer programming, but they are used interchangeably in mathematics books.
* The update equation looks simple, but all practical problems have more than 2 dimensions that we are interested in exploring.  Even for the most simple model $m$, we will want a (1) distribution of posterior beliefs for (2) possible $A$. That's already a 2D plot. but then, the $A$ can be 1,2,n dimensional, we can have different observation scenarios or multidimensional observations, and a variety of prior beliefs to worry about. And then, there is the variety of possible $models$. How do you visualize that?
* It seems that, in paper-and-pencil maths, symbols are often aliased and shortened to save on hand movements. Modern practical computer science has long demonstrated that such savings are counter-productive.

And the most confusing of all -- how does one build the `likelihood function`? Where does it come from? This is not specified in the bayes update equation at all, yet when doing exercises, one is expected to just magically come up with a correct one.

For the longest time I thought that this is something that needs to be derived from the Bayes' equation itself. This is how you solve all the other problems in the school, right? The teacher told you that $F=m*a$ so that's all that you should need. . . . actually, no. Bayes' theory tells you a true nothing, zero, nil, about the model $M$ of the world. This you have to invent separately. Only after you are done inventing, Bayes can tell you something about how good your model $M$ is. This is a true stunner for a high-school children who, up to this moment, were lead to believe that there exists only one correct solution to all problems.

Amazingly, despite all the talk about probabilities, coin tossing and chances of getting cured by a new medicine -- the likelihood function, and the value of the likelihood function, as well as prior and posterior beliefs, have no random chance in them at all. In bayesian thinking, some things are hidden, but no things are random!

The only place where randomness is allowed is the plant. This randomness can be modelled in the model.

---

OK, so ranting over, let me try to clarify some of these things.

> Warning: Over the course of this section, I will rewrite the classic equations a couple of times. Do not call me out on the blatant fact that I use different symbols for the same thing across the first part of this article. This is necessary to make my point.

**1. Reality check.**

To ground the concepts in some tangible reality, let's consider that we have a **plant**.

Plant is just a name for some object, or process, that has inputs and outputs. It does not need to be a manufacturing plant, or like a greenhouse plant. All that it does is that it exists, has inputs, and outputs:

>>> Drawing here <<<

For now, we will only be concerned with the plant's outputs.

This plant produces data $\{ D \}$, which we can observe. The process of producing instances of $D$ can be approximated and described by a $model$ $M$. The model $M$ describes the plant somehow, but at this point we should be clear that there can be more than one good model for that plant (even tough most models will be useless).

Let's suppose that we have a model $M$ of the plant, that takes a hidden(latent) parameter $\phi$. It could be then said that the data $D = M(\phi)$.

We'd like to know what the true $\phi$ is, but we cannot be sure. We cannot measure $\phi$ directly. The $M$ can have a probabilistic nature to it, in sense that depending on $\phi$ it can produce given $D$ more or less often.

Still, we are allowed to have suspicions and beliefs about what the true $\phi$ is. We can observe the events $D$ that happen with $M$'s and $\phi$'s contribution.

**2. Use the reasoning to figure out $\phi$**

For a start, let's replace $A$ and $B$ with model parameter $\phi$ and data(observation) $D$:

$$p( \phi | D) = \frac{p(D|\phi)p(\phi)}{p(D)}$$

Next, let's rewrite the bayes' update equation *slightly* differently (this is not the end of the re-writing):

$$p( \phi | D) = \frac{p(D|\phi)}{p(D)}\times p(\phi)$$

Compared to the original form, this already gives you a better hint: you can get posterior belief, $p( \phi | D)$, from the prior $p(\phi)$, modified by an "updater term", $\frac{p(D|\phi)}{p(D)}$.

Ok, so what about this updater term seems to be so difficult?

Wait. Let me clarify the concepts one by one. And there is quite a bit to untangle, before the road becomes straightforward for us.

**Untangle the concepts of Belief, Probability, and Sample**

Let's take in a new concept: that `belief` is not the same as `probability`, and both are distinct from `sample`.

Here are some hints as to how to separate the two:

* `Belief` is something that you hold in your head. `Probability` is a property of the world. Since there exists an impenetrable epistemological barrier between your "inner" and "outer" world, these things are already distinct.
* Things do not happen because of `belief`, nor because of their `probability`.
* Things happen inside the `plant` according to the plant's model and parameters. The plant's model is $M$ and it's parameter is $\phi$. We can observe data $D=f(M, \phi)$.
* `Belief` is something that we can change ourselves. `Probability` is not something that we can change.
* `Probability` we can estimate from frequency of observation. It follows that for things that are never observed, we cannot talk about their probability.
* `Belief` we can assume, virtually out of nothing. We can then either hold oto that belief, or update it. We can update it, for example, arbitrarily(without any reasoning), or using some rules. For example, using Bayes' update function (maximum amount of reason).
* Estimate of the value of `probability` can be a function of assumptions, including `beliefs`. Updates of `belief` can be a function of `data`.
* `Probability` and `belief` seem to have the same unit, "fraction of $\Omega$" -- where $\Omega$ is "all that there is, all possibilities". Maybe this is the reason why historically they both have been noted as $p()$. This is a poor reason. I propose that we give them a units.
* I propose that, to make it easier to keep a distinction between `probability` and `belief`, they carry distinct units.

Here, I propose the following units:

I will be using unit of `Rey`, or R for belief, that certain statement is true.

I will be using the symbol $\Omega$ for probability on that a certain parameter, event, or statement, or data point $E$ will happen in the future, or is happening while we do not see it happening. Note again, that this is very different from having a completely certain data set that shows that "$E$ has happened 70% of the time".

While we are at it, we can also add an unit for that observed dataset, $\{ D \}$. The reason for a separate unit here is that the set of all possibilities, $\Omega$ is infinite, while the dataset $\{ D \}$ is a finite **sample** from $\Omega$. Hence, it is **not true** that if we have a dataset that shows "$D$ has, so far, happened 70% of the time" is equal to "$D$ is 70% of $\Omega$". Instead, let's define a new unit, [$S$] to describe the prevalence of $E$ in dataset $\{ D \}$.

To summarize:

Given that, for us, $E$ could mean either $\phi$ or $D$, we have:

0.7[R]=700[mR] means "My belief is such that I am 70% sure that a specific value of $\phi$ is true. I leave the 30% to beliefs that some other value of $\phi$ can be true."

0.8[$\Omega$]=800[m$\Omega$]  means "In this chance process, If I sample forever, I will get $D$ 80% of the time. I will get something else 30% of the time".

0.9[$S$]=900[m$S$] means "In this dataset, which is a sample of $\Omega$, event $D$ occurred 90% of the time".

Again, although [R], [$\Omega$], and [$S$] seem to be mathematically interchangeable and can be expressed in percent, or fraction of a whole -- semantically they are distinct, as they refer to different concepts.

Having these insights, I propose a following notation.

## The stage star $b(\phi)$

Let's use function symbols, $f(something)$ :

* $b$ for belief, both prior and posterior,
* $l$ is for likelihood,
* $m$ for marginal probability.

Let's use the variable symbols, $something$:
* $\phi$ for a certain model parameter that belongs to a set of considered model parameters $\{ \phi \}$
* $D$ for a datapoint that already occurred, and we have it in a dataset $\{ D \}$; the dataset $\{ D \}$ has been sampled from a population $\Omega$

It follows that,

The **new belief** about parameter $\phi$, that takes into account the **new information** from observing a datapoint $D$, is noted as $b(\phi|D)$,

And that $b(\phi|D)$ **can be calculated** as coming from old(prior) belief about $\phi$, noted as $b(\phi)$

And that this calculation involves evaluating a Bayesian modifier term, $\frac{l(D|\phi)}{m(D)}$:

$$b(\phi|D) \leftarrow \frac{l(D|\phi)}{m(D)} \times b(\phi)$$

## The ugly duck $m(D)$

The marginal probability $m(D)$ is also sometimes called "evidence". It is calculated by summing (or integrating) the likelihood over all considered values of $\phi$, weighted by the prior belief in $\phi$.

$$m(D) = \sum_{\phi}{l(D|\phi)b(\phi)}$$

But wait!!! In the above equation, the $\phi$ is not the same $\phi$ as in the previous equation! If you didn't know that, I do not blame you.

Instead, here $\phi$ is to say "for all $\phi$s". Because of this, one is not allowed to substitute this expression for $m(D)$ into the bayes' update equation from the previous section -- at least, not using regular algebraic rules as learned in high school : that would create a notation conflict!

In my school times, changing the meaning of notation between different classes was the single most confusing obstacle to quick assimilation of new concepts. If I learned something in the chemistry class, I had to forget about it before going to the Physics class, or else I was in for trouble!

Be advised that for a child's mind, like for the early computers, all symbols and variables are "global". Have you seen that viral video where a 3-year old girl says that a black man ate the cookies? She did not learn to remap the concepts for political correctness when being filmed. She merely rehashed the concepts she heard from the people that surround her.

For adults, switching the context is possible, but still taxing.

Changing the meaning of notation in a middle of doing a school problem is a good recipe for catastrophe. This is in no small part due to that the children are plain not afforded the time and slowdowns that it takes to create correct concept remapping in their heads. They are expected to solve a problem in under 5 minutes, or fail. You tell me what happens to most of us.

Let's make things easier to learn, by using a clearer, non ambiguous notation.

In order to note that the $\phi$ in the following equation has a "set of $\phi$" meaning, rather than just "single scalar $\phi$" meaning, let's note it as $\{\phi\}$. The symbols $\{,  \}$ are used in secondary schools to denote closed sets. That hints the student that he must consider the entire set of $\phi$s and not just a single $\phi$.

Moreover, we can prepare the same person for using list comprehension semantic from Python (and hence, make Python easier for this person), by using notation like "for $\alpha$ in $\{\phi\}$" or "for $\alpha \in  \left<\phi\right>$":

$$m(D) = \sum_{\alpha \ \in \ \{\phi\}}{l(D|\alpha)b(\alpha)}$$

Now we are safe to write that

$$b(\phi|D) \leftarrow b(\phi) \times \frac{l(D|\phi)}{  \sum_{\alpha \ \in \ \{\phi\}}{(l(D|\alpha)b(\alpha))} }$$

The veterans will notice how now, no symbol aliasing occurs. Every symbol has unique semantic meaning.

Moreover, we can be much more clear about where the specific numbers needed are to come from. Only now, having this unambiguous expression, one is equipped to attempt to solve practical problems.

> Warning: In the following part of the article, I will not be changing the notation any more, so feel free to call me out on any errors.

---

## Prior $b(\phi)$

$b(\phi)$ is the initial, or prior **belief** about one of the possible $\phi$s.

$b(\{\phi\})$ is the initial, or prior **belief** about all of the possible values for $\phi$s, and is a vector, or a set: that is, there is a new value $b(...)$ for each $\phi$ from the set of $\{ \phi \}$s

Values for $b(\{\phi\})$ come truly from outside of the dataset and model. In other words, one has to start with some beliefs, something based on external knowledge of the problem at hand.

At this point, most books go into depths about the challenges of holding an "informative" or "uninformative" prior, and give (often unclear) experimental examples on how do they affect the result.

Here I hope to make the examples clearer, by the virtue of using the unambiguous notation developed in the previous section.

For example, we can have, for $\{\phi\}$ = [0.1, 0.5, 0.9], $b(\{\phi\})$ = [0.1, 0.8, 0.1]. This means that our belief is that $\phi$ is most likely 0.5, but we also allow for a total doubt of 20% that it can give way to $\phi$ being 0.1 or 0.9.

Importantly, this distribution does sum up to 1, that is $\sum_{\alpha \ \in \ \{\phi\}}b(\alpha) \ \ = 1$

One can start either with an "uninformative" prior or an "informative" prior. "uninformative" priors are fine if there is enough experimental data to come by; however, if the data is scarce, and there is something that we already know about the problem (e.g., someone told us, or we made a related but not identical experiment), then would be a mistake not to incorporate it into our thought process.

---

## Posterior $b(\phi | D)$

$b(\phi | D)$ is the posterior **belief** after update.

For simple problems, we often talk not about different $D$s, but rather just one discrete $D$. Obviously, data sets $\{ D\}$ will also enter the fray as we become more proficient.

It can be important to note that there is a difference between:
* $b(\phi | D)$ -> Scalar
* $b(\{\phi\} | D)$  -> vector
* $b(phi | \{ D\})$  -> vector

As all of these involve a vector computation or a loop of some kind.

Having that, it is even more clear to see that $b(\{ \phi \} | \{ D\})$ denotes and evaluates to a 2D array of numbers.

Alternatively, we could talk about updating our belief for one specific $\phi$ depending on what $D$ we get. In other words, building a closed-form function. Although this view can be useful, this is not what most examples in the literature are about. $D$ is most often a given constant.

$b(\{\phi\} | D)$ over a range of $\phi$, evaluates to a 1D numeric vector, and this is what you typically want to get to: a description of the final belief about what the latent $\phi$ could be. In other words, an indication of "what should you believe about the different possible $\phi$". This distribution does sum up to 1.

For example, let's say, that after having our prior beliefs $b(\{\phi\})$ = [0.1, 0.8, 0.1] of what the latent value of $\phi$ could be, we observe new data point, $D$. Having this new data point, we update (it doesn't yet matter how) our belief for values of $\{\phi\}$=[0.1, 0.5, 0.9] to be new $b(\{\phi\} | D)$=[0.45, 0.45, 0.1]. Meaning, that we still believe that $\phi$ value of 0.9 is incredible, but now give equal credibility to values $\phi$=0.1 and $\phi$=0.5.

To summarize, using the $"|D"$ part of the notation is there to symbolize that this is about the updated belief, given the data point $D$. Not that we have a sweep of datapoints, and not that we distribute our belief across different $\phi$.

---

## The Likelihood function $l(D|\phi)$

$l(D|\phi)$ is the "likelihood function".

"Likelihood" itself, is a word that typically doesn't really tell you much, because the meaning used here is quite different and distinct from the common-language synonyms of "likelihood". Here, by "likelihood" we do not mean any of "frequency", "chance", "odds", "feasibility", "plausibility" e.t.c. We mean something quite specific here.

Let's attach concrete meaning to the word "likelihood".

Here, "likelihood" it means "the function, and the values of probabilities that single point(or single batch) of data $D$ has been generated by the $model$ $M$ with a given parameter $\phi$".

Likelihood values carry the unit of probability, $[\Omega]$

Notably,

* A value array $l(D|\{\phi\})$ does not need to sum to 1.0 as in the case of belief distribution.
* the individual values of $l(D|\phi)$ for any discrete $\phi$ must still be in the range of (0, 1), as it should be with probability.

For a contrived example, we can have $l(D|\{ \phi \})$ = [1.0, 0.1, 0.5] for $\{ \phi \}$ = [0.1, 0.5, 0.9] and data $D$ = [0].

This means that:
* The data value "0" is reliably always generated by the $model$ $M$ when the $model$'s $\phi$==0.1.
* if the $\phi$==1, then this data would only be generated rarely, approx. 10% of cases.
* if the $model$'s $\phi$=0.9, then the data value "0" is still generated approximately half of the time.

When sampling infinitely from the model $M$ having the parameter $\phi$, we cannot get a $D$ population fraction bigger than 1.0$[\Omega]$ or smaller than 0.0$[\Omega]$.

However, the sum of all $l(D|\{ \phi \})$, or $\sum_{\alpha \in \phi}{p(D|\alpha)}$ can be more or less than 1.00

Moreover, if we observe a single data point $D$=[0], then we still cannot be sure if the true value of model's $\phi$ was 0.1, 0.5, or 0.9. All possible values of $\phi$ are consistent with getting $D$=[0] sometimes.

Note that I have used the word `probability that` rather than `belief that`. This is because for likelihood, there is no guessing of any kind involved, and there is nothing latent(hidden). Instead, given a datapoint, and given one (or a list) of model parameter values, we calculate the already-happened chance that the data, as seen, has happened. We can calculate this chance for all possible hidden parameter values, irrespectively of our belief in them. This is kind of like a grid search, or grid view. Us doing this calculation does not favour any $\phi$ out of the set of $\{ \phi \}$ (yet) and the result does not mean that any specific $\phi$ is the right one.

Obviously, in order to perform this calculation, we need a model of the world, and a kind of that uses these hidden parameters and is conductive to our calculation. Here much of the trouble and effort of the user of bayes' rule comes in. Many treatises on how awesome the bayes rule is will not help you with this.

It is up to the user to construct a model, make sure that the model is representative of the reality, and that it is conductive to the likelihood function value evaluation.

Let's take a look at two most common, basic models, and how is the likelihood function constructed for them. The examples listed below are NOT to say that this is the only way the likelihood function can be constructed!

## Example model of coin toss, and constructing it's likelihood function.

The classic coin-toss toy model is a good one, and widely applicable and extensible, if explained correctly.

### Model description

Here it is so that you don't have to go back to the book.

Let's say that the result of the coin toss can be 0 or 1. Our model approximates the real coin by ignoring the possibility of the real coin landing on the edge.

---

Proposition: biased coin gives "1s" more often.

---

Let's say that the coin could be biased(that is, unfair) and we describe this bias with a parameter $\phi$. We do not know, and cannot know directly what the true $\phi$ is. What we do know is that the $\phi$ can be anywhere from 0.0 to 1.0, with a 'fair' coin being at $\phi \equiv$ 0.50.

* 0.10 means that the coin is very unfair towards only giving zeroes,
* 0.90 would mean that it is very unfair towards only giving ones.
* 0.50 means that it gives 0 in 50% times, and 1 in 50% of times, that is, the coin is fair

### The Likelihood function for the model

To compile this description into something computable, we can say that our model for probability of getting a 1 is $l(D  \equiv 1 | \phi)= \phi$.  Symmetrically, the model of probability of getting a zero, is $l(D \equiv 0 | \phi)=(1-\phi)$

This likelihood function is not something that comes out of the world. It does not come from the bayes rule. It is a new construct that we have created to link our suspected coin bias $\phi$ with the probability of getting an observation $D$. We have created a $model M$. We are using the model $M$ to approximate and describe the real coin.

Note that there is no belief involved here, only assumptions; we assume that $model M$ is an approximation of a real coin.

### Prior belief

We can give our prior belief that the coin is fair, and our disbelief that it is unfair. We can do it by setting, for a possible values of $\{\phi\}$ = [0.1, 0.5, 0.9],  values of initial belief as $b(\{\phi\})$ = b([0.1, 0.5, 0.9]) = [0.1, 0.8, 0.1]

### Calculations

Putting in some concrete numbers, let's say that we believe that $\phi=0.5$ (perfectly fair) and then we toss the coin, and get $D\equiv1$. What was the likelihood to get such a result? $l(D \equiv1 | \phi  \equiv 0.5) = 0.5$.

At this point, we can end our lame discussion -- the coin is fair, the chance of getting a one was 50%, so everything is fine, right?

That's correct. However, what if we do not fully believe that the coin is fair? what if we suspect that the coin is actually biased towards zero?

Let's compute what was the chance of getting a 1, if the latent parameter $\phi$ was 0.1: $l(D  \equiv 1 | \phi  \equiv 0.1) = 0.1$.

So, **If** the coin was heavily biased towards zero, then we still could get a 1, 10% of the time, and getting a result of "1" in one coin toss is not very surprising.

There is seemingly nothing to worry about, except for that we did not make any progress on discovering the true value of $\phi$,  even tough we have made an indirect observation of it, through observing the $D$.

All that we know so far is that the likelihood of observing the result, depends on the parameter in a model $M$ of the plant $P$ that generated that result. To say this, is to say nearly quite nothing.

### No result?

What we really care to know, is what to believe about $\{ \phi \}$ -- is the real value closer to 0.5 or to 0.1? So far we have nothing from reason to go by to believe either of these things. Or do we?

Here is the first time when the bayesian update comes in. And, this is what the books on bayesian reasoning fail the hardest at. They start with discussing prior belief, posterior belief, and worry about how the prior affects the posterior, and why having a prior belief is or is not a good idea. And how to convince other people to your priors. They droll about informative or uninformative priors. Prior this, prior that.

However, all this fails because the Bayesian update is truly useless unless you explain and understand the likelihood function first.

Again: Be aware that the likelihood function DOES NOT COME FROM THE BAYES EQUATION. It comes from the model of the plant!

Before we get to the bayesian belief update, please muster your patience, and take a minute and think about other possible models of likelihood for coin toss-- even if they are implausible. Or for any other model of other world phenomena that you care about. Another classic problem in this category is looking for disease in population, using an imperfect test. I am sure you have heard about it.

### Why bother?

In the previous section, we have created and used a "forward model" of the world.

The reason we do it this way,

is that the "forward models" that is, casual models (where the principle of action-reaction is held)

of this kind are relatively easy to make for a very wide variety of real-world phenomena.

## Example model with linear regression, and constructing its likelihood function.

I bet that you are anxious to hear if bayesian reasoning can be used to any more than coin tosses, or figuring out how if taking a coronavirus test makes sense or not.

Yes, it can. Here is an example on how would you treat a linear regression problem, of the kind:

Proposition: Taller people are heavier.

Question: What is the proportionality coefficient?

Clarification: In Bayesian belief terms, what should we believe about various propositions for the value of proportionality coefficient?

### Model description

$\mapsto$ model input: height $h$

$\mapsto$ model output: weight $w$

$\mapsto$ model parameters: proportionality coefficient $\phi_{prop}$, and the uncertainty descriptor $\phi_{\sigma}$. In other words, we have two latent parameters, not one.

$\mapsto$ Plant's model of uncertainty: random variable $N(0,\sigma )$

$\mapsto$ The complete model for predicting the weight for a person of height $h$ is $\begin{equation}w = h * \phi_{prop} + N(0,\phi_{\sigma} )\end{equation}$

$\mapsto$ nose-free version of the model -- that is, if we set noise to zero, we get weight a model $w = h * \phi_{prop}$

### Likelihood function for linear regression

Here is the critical bit. We construct the equation for the likelihood function of data point $D_{weight}$ happening, given the pair of ($\phi_{prop}$, $\phi_{\sigma}$).

Let's say that the likelihood, or probability of this data point happening is inversely proportional to the distance of the data point value from the model's predicted noiseless value. Hence we want something like

$ l( D_{weight} | (\phi_{p}, \phi_{\sigma})) \ \ \propto \ \  1/distance$

Here are some proposals on how this could look like.

Distance, is simply the difference between the model's predicted, noiseless $w$ and the observed $D_{w}$. For clarity of notation, let's use symbol delta defined as $\Delta = D_{w} - h * \phi_{prop}$

One possible distance metric is :

$_{proposed}distance = abs ( \Delta ) $

We can also use a square of difference, which has the nice property that it weighs large distance more. (It also has a couple of other nice properties):

$_{proposed}distance = \Delta^{2} $

We also want to weight the distance by the amount of uncertainty.
* If the model certainty is high ($\sigma$ is small), then the "improbability" due to distance will be higher.
* If the model certainty is low ($\sigma$ is large), then the probability is higher even at a distance.

We account for this by "weighing" the distance by $\sigma$:

$_{proposed}weightedDistance =  (\frac{\Delta}{\phi_{std}})^{2}$

One more property that we need, before we can get to the likelihood, is that the likelihood for any prospective parameter must evaluate to between 0..1. For that, we need to get a bit creative . . . or "inspired".

Here's a proposed likelihood function that has all the properties required above, sourced from  https://en.wikipedia.org/wiki/Bayesian_linear_regression#Model_setup

$ l( D_{weight} | (\phi_{prop}, \phi_{\sigma})) = \sigma * \exp(-\frac{1}{2\sigma^{2}} \Delta^{2}) $

This function has this nice property that the maximum value is 1.0, quickly decays towards zero as the $\Delta$ increases, and works nice with $\Delta$ weighted by $\sigma$.

This specific function also has the property that it integrates to 1.00 for $\Delta$ of $(-\infty , +\infty )$ -- but we really do not require this property to be there in the likelihood function. As long as you choose a function that outputs something between 0 and 1, it's fine. To see that this is the case, recall that we are talking about the probability of data given the model parameter. There can be more than one more model parameter that will often (or always) produce a given datapoint value. That's fine. We will make the probabilities in the likelihood compatible with beliefs using a normalizing term - "Evidence".

Again, let me be clear that the above presented likelihood function, is merely an example function that happens to have desirable properties. It is by far not the only function that exhibits this properties! For example, you can have a process that does not exhibit gaussian uncertainty like $\propto \exp(-\frac{1}{x^{2}})$. That's fine for Bayesian reasoning!.

Now, that you see that it is possible to build a likelihood function, suitable to describe the problem of "what should I believe..." in bayesian terms for a regression problem, you can begin to believe that it is possible to construct such a function for a very wide variety of problems!

We are not done yet. There is one more hoop to clear before we can get to the posteriority: The Evidence function.

## Evidence, $m()$

The marginal probability $m(D)$ is also sometimes called "evidence".

Why marginal? this word is not to be confused with things like "unimportant", "extreme", or "bad". The word "marginal" is just a code-word that came from the depths of history. One used to write the results of an experiment (when taking the data) in a table (on paper!). Then, came the time to analise the data. Since copying the numbers by hand is time consuming, one would use the margin of the page to squeeze in additional scribblings. Hence, these computed numbers are "marginal".

Why "evidence"? I am not sure, but what I do understand is that again, the name-word, code-word "evidence" does not have anything to do with common-sense meaning of that word. It does not mean "certainty", "proof" , "confirmation", "verification", "display", "demonstration", e.t.c. Here we are only using this word as a name for a certain operation.

As hinted above, the value of the marginal probability, $m(D)$ is calculated by summing (or integrating) the likelihood function of $D$ for all of the considered values of $\phi$, weighted by the prior belief in $\phi$.

$$m(D) = \sum_{\alpha\in\{\phi\}}{l(D|\alpha)b(\alpha})$$

What we get in effect, is a scaling, or normalization term that makes the units and scale of belief and likelihood match.

In the original formula, the same thing is denoted as $p(D)$. Which is triple as confusing because (a) requires an integral or summation over all $\phi$, (b) it integrates to more than 1.0 and hence (c) it has none of the properties of the other $p()$'s. Uhh. (?!?!?).

If you are confused, I do not blame you.

Instead, I propose that it would make much more sense never to write $p(D)$ nor $p(B)$, but rather, teach what the marginal probability function really is, right away: that $m() = m(D,l(),\{\phi\}, b(\{\phi\}))$.  Sounds complicated? It's tedious to write, but it is not complicated. (That's why people `in the know` shorten it). Let's see an example:

Here is an example how to perform this calculation. For our coin toss example, we have:

$m(D \equiv 1, l(),\{\phi\}, b(\{\phi\}) ) = \ldots$

$\ldots \sum_{\alpha\in\{\phi\}}{l(D \equiv 1|\alpha)b(\alpha}) = \ldots $

$ \ldots l(D \equiv 1 | \phi \equiv 0.1)b(\phi \equiv 0.1) \ldots$

$ \ldots + l(D \equiv 1 | \phi \equiv 0.5)b(\phi \equiv 0.5) \ldots$

$ \ldots + l(D \equiv 1 | \phi \equiv 0.9)b(\phi \equiv 0.9) \ldots$

$ = \ldots \\ $

$ \ldots 0.1*0.1+0.5*0.8+0.9*0.1 = 0.01 + 0.4 + 0.09 \ldots \\ $

$ \ldots  \large{= 0.5}$

So, the marginal probability of the "1" happening, under current belief system, is 0.5.

Kind of anticlimactic?

Wait until you see what happens when the prior beliefs were different (unbalanced prior).

For example, for $b(\{\phi\})$ = [0.8, 0.1, 0.1] we get $m(D \equiv 1, l(),\{\phi\}, b(\{\phi\}) )$ = 0.22 . In other words, if our prior belief was that the coin is biased towards zero, then the "marginal probability" (Evidence!) of getting $D \equiv 1$ is lower!

And then, for $b(\{\phi\})$ = [0.1, 0.1, 0.8], $m(D \equiv 1, l(),\{\phi\}, b(\{\phi\}) )$ = 0.78

For "uninformative" prior belief of $b(\{\phi\})$ = [0.33, 0.33, 0.33], $m(D \equiv 1, l(),\{\phi\}, b(\{\phi\}) )$ = 0.50 again.

How come?

If this surprises you, you are in a good company. The surprise comes from the historical fact that the "marginal probability function" has the word "probability" in it. It would seem that the probability of something happening depends on our belief about it???

Alas, this is not the case. "Marginal Probability" or "evidence" is merely a scaling factor that we need to apply to the likelihood, in our full equation for update of the prior belief:

$$b_{\phi, updated} \ = \ b(\phi|D) \leftarrow  b_{\phi, prior} \times \frac{l()}{m()} \ = \ b(\phi) \times \frac{l(D|\phi)}{  m(D,l(),\{\phi\}, b(\{\phi\})) } \ = \ b(\phi) \times \frac{l(D|\phi)}{  \sum_{\alpha \ \in \ \{\phi\}}{(l(D|\alpha)b(\alpha))} }$$

Hence, it is much more enlightening to say that the "Bayes Factor" -- the ratio $\frac{l()}{m()}$ -- tells us how we should modify our prior belief, given the data point $D$.

This factor can be less than one, or more than one. It could be close to zero if $D$ is unlikely, or it could be tending to infinity if our prior belief was very very low. The "Bayes Factor" is composed of the interplay between the likelihood function and our prior belief about all possible $\{ \phi \}$. Seeing it this way dispels any magic about it.

## Finally, the Bayesian update

Phew.

After all this introduction -- which is not really introduction, it is THE meat that should be taught in school in the first place -- we can get to the bayesian method for updating beliefs:

$$b(\phi|D) \leftarrow b(\phi) \frac{l(D|\phi)}{  \sum_{\alpha \ \in \ \{\phi\}}{(l(D|\alpha)b(\alpha))} }$$

See the next chapter for a demonstration of this method in action.

```
#hide
```

```

```


# 鍛鍊你的資料分析力 | Python 資料分析

>  資料分析套件快速入門

郭耀仁 <yaojenkuo@datainpoint.com> from [DATAINPOINT](https://www.datainpoint.com/)

## 大綱

- 資料分析與管線流
- NumPy 快速入門
- Pandas 快速入門
- Matplotlib 快速入門

## 資料分析與管線流

## 資料分析的一切都與管線流息息相關

圖片來源: <https://r4ds.had.co.nz/>

## 每個環節的意涵

- Import：從常見來源將資料載入分析環境
- Tidy/Transform：以適當的資料結構整併內容與轉換樣式
- Visualize：探索分析資料的形狀、相關、組成與趨勢
- Model：預測或挖掘資料的隱含特徵
- Communicate：向產品、行銷與管理團隊精準且有效地傳達分析洞察
- Program：利用程式語言掌握管線流

## 每個環節都有對應套件支援

- [NumPy](https://numpy.org/) 可以支援 Import、Tidy/Transform 與 Model
- [Pandas](https://pandas.pydata.org/) 可以支援 Import、Tidy/Transform 與 Visualise
- [Matplotlib](https://matplotlib.org/) 可以支援 Visualise、Model 與 Communicate

## 確認分析環境能夠使用這三個套件

```python
import numpy as np
import pandas as pd
import matplotlib as mpl
```

## 檢視這些第三方套件的版本資訊

```python
print(np.__version__)
print(pd.__version__)
print(mpl.__version__)
```

    1.18.4
    1.0.3
    3.2.1

## NumPy 快速入門

## 簡介 NumPy 套件

> NumPy，Numerical Python 的簡稱，是使用 Python 進行科學計算的第三方套件，創造了一個稱為 N 維陣列（ndarray）的類別，透過 N 維陣列，可以將 Python 從一個泛用（general purposed）程式語言轉變成一個科學計算（scientific computing）程式語言，並且有豐富的統計、線性代數與隨機的函式。

## 範例資料：COVID-19 每日報告

資料來源：<https://github.com/CSSEGISandData/COVID-19/tree/master/csse_covid_19_data/csse_covid_19_daily_reports>

```python
import datetime

def get_latest_daily_report():
    """
    This function returns the latest global daily report from https://github.com/CSSEGISandData/COVID-19 and its file date.
    """
    latest_date = datetime.date.today()
    day_delta = datetime.timedelta(days=1)
    fmt = '%m-%d-%Y'
    while True:
        try:
            latest_date_fmt = latest_date.strftime(fmt)
            csv_url = "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/{}.csv".format(latest_date_fmt)
            daily_report = pd.read_csv(csv_url)
            print("載入了 {} 的每日報告。".format(latest_date_fmt))
            break
        except:
            latest_date_fmt = latest_date.strftime(fmt)
            print("尚未有 {} 的每日報告。".format(latest_date_fmt))
            latest_date -= day_delta
    return latest_date, daily_report
```

```python
latest_date, daily_report = get_latest_daily_report()
```

    尚未有 09-13-2020 的每日報告。
    載入了 09-12-2020 的每日報告。

## N 維陣列可由資料框取出

```python
confirmed = daily_report['Confirmed'].values
type(confirmed)
```

    numpy.ndarray

## N 維陣列提供兩個好用的功能

- 向量化（Vectorization）運算、有時也稱為元素級別（Element-wise）運算
- 布林索引（Boolean indexing）

## 向量化運算

```python
deaths = daily_report['Deaths'].values
recovered = daily_report['Recovered'].values
active = confirmed - deaths - recovered # Vectorization
```

## 布林索引

```python
print(confirmed.size)
print(confirmed > 100000)                 # Creating a boolean array
print(confirmed[confirmed > 100000].size) # Boolean indexing
```

    3954
    [False False False ... False False False]
    65

## 試著使用內建資料結構 `list` 完成前兩個任務（藉此體驗 N 維陣列的便利性）

1. 計算治療中個案數；
2. 挑選確診數超過 10 萬的觀測值。

```python
# 計算治療中個案數
confirmed = list(daily_report['Confirmed'].values)
deaths = list(daily_report['Deaths'].values)
recovered = list(daily_report['Recovered'].values)
```

```python
# 挑選確診數超過 10 萬的觀測值
confirmed = list(daily_report['Confirmed'].values)
```

## 除了 N 維陣列，NumPy 還附帶豐富的函式

- 通用函式
- 聚合函式
- 隨機函式
- 線性代數函式
- ...etc.

## 暸解更多 NumPy 提供的功能

[NumPy User Guide](https://www.numpy.org/devdocs/user/index.html)

## Pandas 快速入門

## 簡介 Pandas 套件

> Pandas，Panel DataFrame Series 的簡稱，是在 Python 中分析表格資料的第三方套件，創造了稱為索引（Index）、序列（Series）與資料框（DataFrame）的類別，透過這些類別，可以讓 Python 在面對文字檔案、Excel 試算表與關聯式資料庫時能夠使用更直覺的觀念操作。

## 將 CSV 文字檔案讀入成為資料框

- 範例資料：COVID-19 每日報告
- 資料來源：<https://github.com/CSSEGISandData/COVID-19/tree/master/csse_covid_19_data/csse_covid_19_daily_reports>

```python
latest_date, daily_report = get_latest_daily_report()
type(daily_report)
```

    尚未有 09-13-2020 的每日報告。
    載入了 09-12-2020 的每日報告。

    pandas.core.frame.DataFrame

## Pandas 提供三種新的資料結構

- `DataFrame`
- `Series`
- `Index`

```python
print(type(daily_report))              # DataFrame
print(type(daily_report['Confirmed'])) # Series
print(type(daily_report.columns))      # Index
print(type(daily_report.index))        # Index
```

    <class 'pandas.core.frame.DataFrame'>
    <class 'pandas.core.series.Series'>
    <class 'pandas.core.indexes.base.Index'>
    <class 'pandas.core.indexes.range.RangeIndex'>

## 使用更直覺的觀念操作資料

- 如何定義「更直覺」？
    - Spreadsheet-like；
    - SQL-like

## 基礎的資料框操作

- 衍生（mutate），新增欄位到資料框中，特別是新欄位與既有欄位具有函式的輸出以及輸入關係；
- 選擇（select），從資料框中依據名稱挑出單個或多個欄位；
- 篩選（filter），依據判斷條件（布林值）從資料框中挑出符合（布林值為 True）的觀測值；
- 摘要（summarise），對欄位進行聚合（Aggregate）的運算將多筆觀測值總結；
- 排序（arrange），對觀測值由小到大（遞增）或者由大到小（遞減）變動排列順序；
- 分組（group by），將欄位依照獨一類別進行摘要。

## 衍生（mutate）

治療中案例數可以用既有的欄位計算而得，常見的定義為：確診數減去死亡數再減痊癒數。

\begin{equation}
\text{Active} = \text{Confirmed} - \text{Deaths} - \text{Recovered}
\end{equation}

```python
active = daily_report["Confirmed"] - daily_report["Deaths"] - daily_report["Recovered"]
active
```

    0        5987
    1        4361
    2       12527
    3         348
    4        1914
            ...
    3949     9717
    3950        1
    3951      216
    3952     1147
    3953     1609
    Length: 3954, dtype: int64

## 選擇（select）

在中括號裡頭輸入欄位名稱可以將資料以 `Series` 外型從資料框中取出。

```python
daily_report["Country_Region"]
```

    0              Afghanistan
    1                  Albania
    2                  Algeria
    3                  Andorra
    4                   Angola
                   ...
    3949    West Bank and Gaza
    3950        Western Sahara
    3951                 Yemen
    3952                Zambia
    3953              Zimbabwe
    Name: Country_Region, Length: 3954, dtype: object

## 選擇（select）

若想要選擇多個欄位，將多個欄位名稱以 `list` 傳入中括號。

```python
multiple_columns = ["Country_Region", "Confirmed", "Deaths", "Recovered"]
daily_report[multiple_columns]
```

      Country_Region
      Confirmed
      Deaths
      Recovered

      0
      Afghanistan
      38641
      1420
      31234

      1
      Albania
      11185
      330
      6494

      2
      Algeria
      48007
      1605
      33875

      3
      Andorra
      1344
      53
      943

      4
      Angola
      3335
      132
      1289

      ...
      ...
      ...
      ...
      ...

      3949
      West Bank and Gaza
      29906
      210
      19979

      3950
      Western Sahara
      10
      1
      8

      3951
      Yemen
      2009
      582
      1211

      3952
      Zambia
      13466
      312
      12007

      3953
      Zimbabwe
      7508
      224
      5675

3954 rows × 4 columns

## 篩選（filter）

在中括號裡頭輸入判斷條件所獲得的布林值 `Series` 可以獲得指定的觀測值。

```python
is_tw = daily_report['Country_Region'] == 'Taiwan*' # 台灣在哪裡
is_tw
```

    0       False
    1       False
    2       False
    3       False
    4       False
            ...
    3949    False
    3950    False
    3951    False
    3952    False
    3953    False
    Name: Country_Region, Length: 3954, dtype: bool

```python
daily_report[is_tw] # is_tw 是一個布林值 Series
```

      FIPS
      Admin2
      Province_State
      Country_Region
      Last_Update
      Lat
      Long_
      Confirmed
      Deaths
      Recovered
      Active
      Combined_Key
      Incidence_Rate
      Case-Fatality_Ratio

      622
      NaN
      NaN
      NaN
      Taiwan*
      2020-09-13 04:30:52
      23.7
      121.0
      498
      7
      475
      16.0
      Taiwan*
      2.090963
      1.405622

## 摘要（summarize）

對 `Series` 呼叫聚合方法，例如加總 `.sum()`。

```python
daily_report['Confirmed'].sum()
```

    28759036

## 分組（group by）與摘要（summarise）

以國家為分組層級，所以在 `groupby()` 方法中傳入 Country_Region 欄位，獲得一個 `DataFrameGroupBy` 類別。

```python
daily_report.groupby("Country_Region")
```

    <pandas.core.groupby.generic.DataFrameGroupBy object at 0x7fa4a8b78898>

## 分組（group by）與摘要（summarise）

指定 `DataFrameGroupBy` 類別的欄位與聚合函式，獲得分組摘要的結果，是一個 `Series`。

```python
daily_report.groupby("Country_Region")['Confirmed'].sum()
```

    Country_Region
    Afghanistan           38641
    Albania               11185
    Algeria               48007
    Andorra                1344
    Angola                 3335
                          ...
    West Bank and Gaza    29906
    Western Sahara           10
    Yemen                  2009
    Zambia                13466
    Zimbabwe               7508
    Name: Confirmed, Length: 188, dtype: int64

## 排序（arrange）

呼叫 `Series` 的 `sort_values(ascending=False)` 方法將摘要結果由大到小遞減排序。

```python
confirmed_by_country = daily_report.groupby("Country_Region")['Confirmed'].sum()
confirmed_by_country.sort_values(ascending=False)[:10] # 顯示確診人數前 10 高的國家
```

    Country_Region
    US              6485214
    India           4754356
    Brazil          4315687
    Russia          1053663
    Peru             716670
    Colombia         708964
    Mexico           663973
    South Africa     648214
    Spain            566326
    Argentina        546481
    Name: Confirmed, dtype: int64

## 除了基礎操作，Pandas 還能夠進行

- 網頁表格內容載入
- 關聯式資料庫操作
- 視覺化
- ...etc.

## 暸解更多 Pandas 提供的功能

[pandas: powerful Python data analysis toolkit](http://pandas.pydata.org/pandas-docs/stable/)

## Matplotlib 快速入門

## 簡介 Matplotlib 套件

> Matplotlib，Matlab Plotting Library 的簡稱，是在 Python 中將資料視覺化的第三方套件。

## pyplot 模組

> 隸屬於 Matplotlib 的繪圖工具，用類似 Matlab 的語法建立視覺化。

```python
import matplotlib.pyplot as plt
```

## 使用流程

- 將資料整理為 `ndarray` 或 `Series` 格式
- 呼叫 `plt.figure()` 展開「畫布物件」、呼叫 `plt.axes()` 展開「軸物件」
- 依照探索需求呼叫「軸物件」的作圖方法
- 依照設計需求添加「軸物件」的元素
- 呼叫 `plt.show()` 顯示圖形（或者 `plt.savefig()` 儲存圖形）

## 將 CSV 文字檔案讀入成為資料框

- 範例資料：COVID-19 時間序列
- 資料來源：<https://github.com/CSSEGISandData/COVID-19/tree/master/csse_covid_19_data/csse_covid_19_time_series>

```python
request_url = "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv"
time_series = pd.read_csv(request_url)
type(time_series)
```

    pandas.core.frame.DataFrame

```python
time_series.head()
```

      Province/State
      Country/Region
      Lat
      Long
      1/22/20
      1/23/20
      1/24/20
      1/25/20
      1/26/20
      1/27/20
      ...
      9/3/20
      9/4/20
      9/5/20
      9/6/20
      9/7/20
      9/8/20
      9/9/20
      9/10/20
      9/11/20
      9/12/20

      0
      NaN
      Afghanistan
      33.93911
      67.709953
      0
      0
      0
      0
      0
      0
      ...
      38288
      38304
      38324
      38398
      38494
      38520
      38544
      38572
      38606
      38641

      1
      NaN
      Albania
      41.15330
      20.168300
      0
      0
      0
      0
      0
      0
      ...
      9844
      9967
      10102
      10255
      10406
      10553
      10704
      10860
      11021
      11185

      2
      NaN
      Algeria
      28.03390
      1.659600
      0
      0
      0
      0
      0
      0
      ...
      45469
      45773
      46071
      46364
      46653
      46938
      47216
      47488
      47752
      48007

      3
      NaN
      Andorra
      42.50630
      1.521800
      0
      0
      0
      0
      0
      0
      ...
      1199
      1215
      1215
      1215
      1261
      1261
      1301
      1301
      1344
      1344

      4
      NaN
      Angola
      -11.20270
      17.873900
      0
      0
      0
      0
      0
      0
      ...
      2805
      2876
      2935
      2965
      2981
      3033
      3092
      3217
      3279
      3335

5 rows × 239 columns

## 資料整理：轉置

```python
id_cols = time_series.columns[:4]
time_series_long = pd.melt(time_series, id_vars=id_cols, var_name='Date', value_name='Confirmed')
time_series_long.head()
```

      Province/State
      Country/Region
      Lat
      Long
      Date
      Confirmed

      0
      NaN
      Afghanistan
      33.93911
      67.709953
      1/22/20
      0

      1
      NaN
      Albania
      41.15330
      20.168300
      1/22/20
      0

      2
      NaN
      Algeria
      28.03390
      1.659600
      1/22/20
      0

      3
      NaN
      Andorra
      42.50630
      1.521800
      1/22/20
      0

      4
      NaN
      Angola
      -11.20270
      17.873900
      1/22/20
      0

## 資料整理：轉換 `Date` 型別

```python
date = pd.to_datetime(time_series_long['Date'])
time_series_long = time_series_long.drop('Date', axis=1)
time_series_long.insert(4, 'Date', date)
time_series_long.info()
```

    <class 'pandas.core.frame.DataFrame'>
    RangeIndex: 62510 entries, 0 to 62509
    Data columns (total 6 columns):
     #   Column          Non-Null Count  Dtype
    ---  ------          --------------  -----
     0   Province/State  19035 non-null  object
     1   Country/Region  62510 non-null  object
     2   Lat             62510 non-null  float64
     3   Long            62510 non-null  float64
     4   Date            62510 non-null  datetime64[ns]
     5   Confirmed       62510 non-null  int64
    dtypes: datetime64[ns](1), float64(2), int64(1), object(2)
    memory usage: 2.9+ MB

## 繪製兩個視覺化

1. 台灣累計確診人數趨勢圖；
2. 台灣每日新增確診人數長條圖。

## 繪製台灣累計確診人數趨勢圖

## 將資料整理為 `ndarray` 或 `Series` 格式

```python
tw = time_series_long[time_series_long['Country/Region'] == 'Taiwan*']
x = tw['Date'].values
y = tw['Confirmed'].values
```

## 呼叫 `plt.figure()` 展開「畫布物件」、呼叫 `plt.axes()` 展開「軸物件」

```python
fig = plt.figure()
ax = plt.axes()
```

## 依照探索需求呼叫「軸物件」的作圖方法

```python
fig = plt.figure()
ax = plt.axes()
ax.plot(x, y)
```

## 依照設計需求添加「軸物件」的元素

```python
fig = plt.figure()
ax = plt.axes()
ax.plot(x, y, label="Taiwan")
ax.set_xlabel('Date')
ax.set_title('Cumulative COVID-19 confirmed cases')
ax.legend(loc='upper left')
```

## 呼叫 plt.show() 顯示圖形

或者 `plt.savefig()` 儲存圖形。

```python
fig = plt.figure()
ax = plt.axes()
ax.plot(x, y, label="Taiwan")
ax.set_xlabel('Date')
ax.set_title('Cumulative COVID-19 confirmed cases')
ax.legend(loc='upper left')
plt.show() # plt.savefig('tw_covid19_time_series.png')
```

## 繪製台灣每日新增確診人數長條圖

## 將資料整理為 `ndarray` 或 `Series` 格式

```python
daily_increase = np.diff(tw['Confirmed'].values, n=1)
```

## 呼叫 `plt.figure()` 展開「畫布物件」、呼叫 `plt.axes()` 展開「軸物件」

```python
fig = plt.figure()
ax = plt.axes()
```

## 依照探索需求呼叫「軸物件」的作圖方法

```python
fig = plt.figure()
ax = plt.axes()
ax.bar(x[1:], daily_increase)
```

## 依照設計需求添加「軸物件」的元素

```python
fig = plt.figure()
ax = plt.axes()
ax.bar(x[1:], daily_increase, label='Taiwan')
ax.set_title('Daily COVID-19 confirmed cases')
ax.set_xlabel('Date')
ax.set_ylabel('Daily Increase')
ax.set_ylim(0, None)
ax.legend(loc='upper right')
```

## 呼叫 plt.show() 顯示圖形

或者 `plt.savefig()` 儲存圖形。

```python
fig = plt.figure()
ax = plt.axes()
ax.bar(x[1:], daily_increase, label='Taiwan')
ax.set_title('Daily COVID-19 confirmed cases')
ax.set_xlabel('Date')
ax.set_ylabel('Daily Increase')
ax.set_ylim(0, None)
ax.legend(loc='upper right')
plt.show() # plt.savefig('tw_covid19_daily_increase.png')
```