# Calculations of supercurrents in a proximitized semiconducting 3D nanowire
The simulations in this notebook generate data that is used for the plotting of the theoretical figures of the paper.

Some simulations are expensive and take ~20hrs on 300 cores. Therefore we often cut up the simulation in parts using the `funcs.run_simulation` function with the `N` argument. The resulting (partial) data is saved in `pandas.DataFrame`s.

To view the resulting data (after joining the parts with `funcs.combine_dfs`, see example just below) see [paper-figures.ipynb](paper-figures.ipynb) for the figures from the paper or [explore-data.ipynb](explore-data.ipynb) to interactively explore the data using sliders.

### Example: joining the data files
```python
# If the simulation is run with:
funcs.run_simulation(lview, func, vals, parameters=dict(**params, **syst_pars),
                     fname_i='tmp/1d_alpha_vs_B_x_{}.hdf', N=20000, overwrite=True)
# You can join the resulting files with:
df = funcs.combine_dfs('tmp/1d_alpha_vs_B_x_*.hdf', 'data/1d_alpha_vs_B_x.hdf')
```

# The Python conda environment that is used
* We used the following Python environment. See [README.md](README.md) for installation instructions.
```yaml
name: python3
channels:
  - conda-forge
dependencies:
  - python=3.6
  - kwant=1.3
  - holoviews=1.8.1
  - xarray=0.9.6
  - pandas=0.20.3
  - pytables=3.4.2
  - toolz=0.8.2
  - hpc05  # This is only needed if you use a PBS cluster with headnode, as on the TU Delft network
```

## Notes
* See [example-toy-models.ipynb](example-toy-models.ipynb) for simpler examples.
* Read the comments!

```python
# Connect the ipyparallel.Client
try:
    # If this can be imported, it assumes you have access to a PBS cluster
    # with a headnode where you started a cluster
    import hpc05
    client = hpc05.Client(profile='pbs', timeout=60)
    print("Connected to hpc05")
except ImportError:
    # Otherwise start an ipyparallel ipcluster locally
    from ipyparallel import Client
    client = Client()
    print("Connected to local engines")
```

```python
dview = client[:]
dview.use_dill()
lview = client.load_balanced_view()
print('Connected to {} engines.'.format(len(dview)))
```

```python
# Make sure this folder is in your $PYTHONPATH of the ipengines
# or set the correct location in the next line (run after all engines are connected)
%px import sys, os; sys.path.append(os.path.expanduser('~/Work/nanowire_current/'))
```

    UsageError: Line magic function `%px` not found.

# Imports

```python
# Related third party imports
import holoviews as hv
import kwant
import numpy as np
import pandas as pd

# Local imports
import funcs

%matplotlib inline
hv.notebook_extension()
```

<link rel="stylesheet" href="https://code.jquery.com/ui/1.10.4/themes/smoothness/jquery-ui.css">

# Creating some wires as example

```python
import sympy
```

```python
sympy.__version__
```

    '1.2'

```python
# Create system with infinite leads
syst_pars = dict(a=8, angle=0, site_disorder=False, holes=True, L=200, L_sc=80,
                 phi=135, r1=50, r2=70, shape='circle', with_leads=True,
                 with_shell=True, with_vlead=True)

syst, hopping = funcs.make_3d_wire(**syst_pars)
kwant.plot(syst);
```

```python
# Create system without leads and bigger SC regions
syst_pars['L_sc'] = 40
syst_pars['L'] = 40
syst, hopping = funcs.make_3d_wire(**syst_pars)
kwant.plot(syst, site_size=0, hop_lw=0.1);
```

```python
# Create system without leads
syst_pars['with_leads'] = False
syst, hopping = funcs.make_3d_wire(**syst_pars)
kwant.plot(syst);
```

# Mean free path calculation
See [mean-free-path.ipynb](mean-free-path.ipynb) for the data processing.

```python
syst_pars = dict(a=8, angle=0, site_disorder=True, holes=False, L_sc=8,
                 phi=135, r1=50, r2=70, shape='circle', with_leads=True,
                 with_shell=False, with_vlead=False)

params = dict(alpha=20, B_x=0, B_y=0, B_z=0, Delta=0, g=50,
              orbital=True, V='lambda x: 0', **funcs.constants.__dict__)

Ls = np.arange(80, 2000, 80)
salts = np.arange(0, 10)
disorders = [0, 10, 20, 30, 40, 50, 55, 60, 65, 70, 75, 80, 90, 100, 110]
mus = np.arange(10, 31)

vals = funcs.named_product(salt=salts, disorder=disorders, L=Ls, mu=mus)
print(len(vals))

def func(val, syst_pars=syst_pars, params=params):
    import kwant, funcs

    # Combine the fixed parameters `params` and the changing
    # parameters `val` to one dict and evaluate the string
    # lambda functions.
    params = funcs.parse_params(dict(**params, **val))

    # Remove the parameters from `params` that belong in `syst_pars`.
    for x in ['L']:
        syst_pars[x] = params.pop(x)

    syst, hopping = funcs.make_3d_wire(**syst_pars)
    smatrix = kwant.smatrix(syst, params=params)

    # Return a combined dictionary with the results and input.
    return dict(transmission=smatrix.transmission(0, 1),
                num_propagating=smatrix.num_propagating(0),
                **val)

fname = 'data/mean_free_path.hdf'
funcs.run_simulation(lview, func, vals, dict(**params, **syst_pars), fname, overwrite=True)
```

# $B_x(I_c)$ combinations of effects and geometries:

```python
syst_pars = dict(a=8, angle=0, site_disorder=False, holes=True, phi=135,
                 r1=50, r2=70, shape='circle', with_shell=True, with_vlead=True)

params = dict(B_y=0, B_z=0, Delta=60, c_tunnel=5/8, V=lambda x: 0, **funcs.constants.__dict__)

Ts = [0.1, 0.5, 1]
orbital_bools = [False, True]
gs = [0, 50]
alphas = [0, 20]
mus = [10, 15, 20, 25, 30]
Ls = [80, 160, 320, 640]
leads = [(True, 8), (False, 400)]
Bs = np.linspace(0, 2, 101)

vals = funcs.named_product(T=Ts, L=Ls, orbital=orbital_bools,
                           g=gs, alpha=alphas, mu=mus, leads=leads, B_x=Bs)

def func(val, syst_pars=syst_pars, params=params):
    import funcs
    val['with_leads'], val['L_sc'] = val['leads']

    # Combine the fixed parameters `params` and the changing
    # parameters `val` to one dict and evaluate the string
    # lambda functions.
    params = funcs.parse_params(dict(**params, **val))

    # Remove the parameters from `params` that belong in `syst_pars`.
    for x in ['with_leads', 'L_sc', 'L']:
        syst_pars[x] = params.pop(x)

    syst, hopping = funcs.make_3d_wire(**syst_pars)

    # Return a combined dictionary with the results and input.
    return dict(**funcs.I_c(syst, hopping, params), **val)

fname = "tmp/I_c(B_x)_no_disorder_combinations_of_effects_and_geometries_{}.hdf"
funcs.run_simulation(lview, func, vals, dict(**params, **syst_pars), fname, 2000)

```

# $B_x(I_c)$ with disorder:
* Main paper figure 4
* Supplementary materials figure 8

```python
ONLY_PAPER_COMBINATIONS = False

syst_pars = dict(a=8, angle=0, site_disorder=True, holes=True, L=640,
                 L_sc=8, phi=135, r1=50, r2=70, shape='circle',
                 with_leads=True, with_shell=True, with_vlead=True)

params = dict(B_y=0, B_z=0, Delta=60, c_tunnel=5/8, V='lambda x: 0', T=100e-3, **funcs.constants.__dict__)

Bs = np.linspace(0, 0.5, 201)

if ONLY_PAPER_COMBINATIONS:
    # These are all the combinations of effects that are showed in Fig. 4 of the main paper.
    vals = [[(0, 0, True, 50, 20, 10, B),
             (0, 0, False, 50, 20, 10, B),
             (0, 0, True, 50, 0, 10, B),
             (0, 0, False, 50, 0, 10, B),
             (0, 0, True, 50, 20, 20, B),
             (0, 0, False, 50, 20, 20, B),
             (0, 0, True, 50, 0, 20, B),
             (0, 0, False, 50, 0, 20, B),
             (7, 75, True, 50, 20, 20, B)]
            for B in Bs]
    vals = sum(vals, [])
    names = ['salt', 'disorder', 'orbital', 'g', 'alphas', 'mu', 'B_x']
    vals = [dict(zip(names, val)) for val in vals]

else:
    # Calculate all combinations of effects (which is much slower.)
    vals = funcs.named_product(salt=np.arange(0, 8, 1), disorder=[0, 75],
                               orbital=[False, True], g=[0, 50], alpha=[0, 20],
                               mu=[10, 20], B_x=Bs)

    # Filter out different salts when there is no disorder.
    vals = [val for val in vals if not (val['salt'] > 0 and val['disorder'] == 0)]

print(len(vals))

def func(val, syst_pars=syst_pars, params=params):
    import funcs
    syst, hopping = funcs.make_3d_wire(**syst_pars)

    # Combine the fixed parameters `params` and the changing
    # parameters `val` to one dict and evaluate the string
    # lambda functions.
    params = funcs.parse_params(dict(**params, **val))

    # Return a combined dictionary with the results and input.
    return dict(**funcs.I_c(syst, hopping, params), **val)

fname_i = "tmp/I_c(B_x)_mu10,20meV_disorder0,75meV_T0.1K_all_combinations_of_effects_{}.hdf"
funcs.run_simulation(lview, func, vals, dict(**params, **syst_pars), fname_i, 2500)
```

# Rotation of the magnetic field in the $xy$-plane
* Supplementary materials figure 10

```python
syst_pars = dict(a=8, angle=0, site_disorder=True, holes=True, L=160, L_sc=8,
                 phi=135, r1=50, r2=70, shape='circle',
                 with_leads=True, with_shell=True, with_vlead=True)

params = dict(alpha=20, B_z=0, Delta=60, g=50, mu=20, salt=7,
              orbital=True, c_tunnel=5/8, V='lambda x: 0', T=100e-3, **funcs.constants.__dict__)

Bs = np.linspace(0, 2, 101)
thetas = [0, 45, 90]
vals = funcs.named_product(B=Bs, theta=thetas, disorder=[0, 75])

def func(val, syst_pars=syst_pars, params=params):
    import funcs
    import numpy as np
    angle = np.deg2rad(val['theta'])
    val['B_x'] = val['B'] * np.cos(angle).round(15)
    val['B_y'] = val['B'] * np.sin(angle).round(15)

    # Combine the fixed parameters `params` and the changing
    # parameters `val` to one dict and evaluate the string
    # lambda functions.
    params = funcs.parse_params(dict(**params, **val))

    syst, hopping = funcs.make_3d_wire(**syst_pars)

    # Return a combined dictionary with the results and input.
    return dict(**funcs.I_c(syst, hopping, params), **val)

fname = 'data/I_c(B_x)_mu20meV_rotation_of_field_in_xy_plane.hdf'
funcs.run_simulation(lview, func, vals, dict(**syst_pars, **params), fname, None, overwrite=True)
```

# Tuning the induced gap

```python
syst_pars = dict(a=8, angle=0, site_disorder=False, holes=True,
                 L=80, L_sc=8, phi=135, r1=50, r2=70, shape='circle',
                 with_leads=True, with_shell=True, with_vlead=False)

params = dict(alpha=20, B_x=0, B_y=0, B_z=0, g=50, orbital=True,
              c_tunnel=5/8, V='lambda x: 0', **funcs.constants.__dict__,)

mus = np.linspace(2, 30, 50)
Deltas = np.linspace(0, 100, 50)

vals = funcs.named_product(Delta=Deltas, mu=mus)

def func(val, syst_pars=syst_pars, params=params):
    import funcs
    import numpy as np
    lead = funcs.make_3d_wire(**syst_pars)[0].leads[1]

    # Combine the fixed parameters `params` and the changing
    # parameters `val` to one dict and evaluate the string
    # lambda functions.
    params = funcs.parse_params(dict(**params, **val))

    # Create the Hamiltonian `ham` at k=0.
    h0 = lead.cell_hamiltonian(params=params)
    t0 = lead.inter_cell_hopping(params=params)
    ham = h0 + t0 + t0.conj().T

    # Find the energies.
    ev = np.linalg.eigvalsh(ham)

    # Return a combined dictionary with the results and input.
    return dict(E_min=np.abs(ev).min(), **val)

funcs.run_simulation(lview, func, vals, dict(**params, **syst_pars), 'data/gap_tuning.hdf')
```

```python
# Plot the gaps and find the `Delta` that is needed for the required `Delta_induced`.
df = pd.read_hdf('data/gap_tuning.hdf')
Deltas = sorted(df.Delta.unique())
mus = sorted(df.mu.unique())
gap = np.reshape(df.E_min.values, (len(Deltas), len(mus)))
max_gap = np.max(np.min(gap, axis=1))
print("Max gap is {} meV".format(max_gap))
```

```python
hv.Curve((Deltas, gap.min(axis=1)), kdims=[r'$\Delta$'], vdims=[r'$E_{gap}$'],
         label=r"$E_{gap}$ over range of $\Delta$'s")
```

```python
Delta_ind = {delta: hv.Path((mus, evs), kdims=[r'$\mu$', ('E_gap', r'$E_{gap}$')])
      for evs, delta in zip(gap, Deltas)}

hm = hv.HoloMap(Delta_ind, kdims=[r'$\Delta$'])
(hm.select(E_gap=(0.15, 0.40)) * hv.HLine(0.25))
```

## Gate simulation with and without disorder $I_c(B_x, V)$
* Main paper figure 5(b)
* Supplementary materials figure 9

```python
# Plot test
syst_pars = dict(a=8, angle=0, site_disorder=False, holes=True,
                 L=640, L_sc=8, phi=135, r1=50, r2=70, shape='circle',
                 with_leads=True, with_shell=True, with_vlead=True)

syst, hopping = funcs.make_3d_wire(**syst_pars)
gate_fun = funcs.gate(syst, 100, 160)
sites = [gate_fun(pos[0]) for pos in [i.pos for i in syst.sites]]
kwant.plot(syst, site_lw=0, site_color=sites, colorbar=False)
```

```python
syst_pars = dict(a=8, angle=0, site_disorder=True, holes=True,
                 L=640, L_sc=8, phi=135, r1=50, r2=70, shape='circle',
                 with_leads=True, with_shell=True, with_vlead=True)

params = dict(alpha=20, B_y=0, B_z=0, Delta=60, g=50, mu=20, orbital=True,
              c_tunnel=5/8, salt=7, T=100e-3, **funcs.constants.__dict__)

Bs = np.linspace(0, 0.5, 51)
Vs = np.linspace(0, 10, 51)
gate_sizes = [160]
disorders = [0, 75]

vals = funcs.named_product(gate_size=gate_sizes,
                           disorder=disorders, V=Vs, B_x=Bs)

def func(val, syst_pars=syst_pars, params=params):
    import funcs
    syst, hopping = funcs.make_3d_wire(**syst_pars)

    # Combine the fixed parameters `params` and the changing
    # parameters `val` to one dict and evaluate the string
    # lambda functions.
    params = funcs.parse_params(dict(**params, **val))

    # Create a potential function for the gate.
    params['V'] = funcs.gate(syst, params['V'], params['gate_size'])

    # Return a combined dictionary with the results and input.
    return dict(**funcs.I_c(syst, hopping, params), **val)

fname = "tmp/I_c(B_x,_V)_gate160nm_mu20meV_disorder0,75meV_T0.1K_{}.hdf"
funcs.run_simulation(lview, func, vals, dict(**params, **syst_pars), fname, 2500)
```

# 1D toymodel $I_c(\alpha, B_x)$
Supplementary materials figure 7

```python
syst_pars = dict(a=10, L=500, L_sc=100)

params = dict(B_y=0, B_z=0, Delta=0.25, g=50, V='lambda x: 0', T=0.1,
              **funcs.constants.__dict__)

alphas = np.linspace(0, 30, 101)
B_xs = np.linspace(0, 1.4, 101)
mus = [0.1, 0.3, 1, 3, 10, 30]

vals = funcs.named_product(alpha=alphas, B_x=B_xs, mu=mus)

def func(val, syst_pars=syst_pars, params=params):
    import funcs
    syst, hopping = funcs.make_1d_wire(**syst_pars)

    # Combine the fixed parameters `params` and the changing
    # parameters `val` to one dict and evaluate the string
    # lambda functions.
    params = funcs.parse_params(dict(**params, **val))

    # Return a combined dictionary with the results and input.
    return dict(**funcs.I_c(syst, hopping, params), **val)

funcs.run_simulation(lview, func, vals, dict(**params, **syst_pars),
                     'tmp/1d_alpha_vs_B_x_{}.hdf', N=20000, overwrite=True)
```

```python

```


# DISTIL Usage Example: CIFAR-10 with Rare Classes

Not all datasets have an even spread of classes among the set of labels. Indeed, a dataset might have only a couple elements that have a particular class as a label. Such classes are considered *rare*, and extra work is required to achieve good model performance on these examples. The typical fix is to provide more data with the rare-class label; however, this issue is complicated by the fact that this data is usually part of a massive unlabeled pool of data. Here, we show how to use DISTIL's implementation of [SIMILAR](https://arxiv.org/abs/2107.00717) to mine these rare examples for labeling.

# Preparation

## Installation and Imports

```python
# Get DISTIL
!git clone https://github.com/decile-team/distil.git
!pip install -r distil/requirements/requirements.txt

import copy
import matplotlib.pyplot as plt
import numpy as np
import os
import sys
import torch
import torch.optim as optim
import torch.nn.functional as F

from torch import nn
from torch.autograd import Variable
from torch.utils.data import Dataset, Subset, ConcatDataset, DataLoader
from torchvision import transforms
from torchvision.datasets import cifar

sys.path.append('distil/')
from distil.active_learning_strategies import SMI, GLISTER, BADGE, EntropySampling, RandomSampling  # All active learning strategies showcased in this example
from distil.utils.models.resnet import ResNet18                                                     # The model used in our image classification example
from distil.utils.train_helper import data_train                                                    # A utility training class provided by DISTIL
from distil.utils.utils import LabeledToUnlabeledDataset                                            # A utility wrapper class that removes labels from labeled PyTorch dataset objects
```

    Cloning into 'distil'...
    remote: Enumerating objects: 3324, done.
    remote: Counting objects: 100% (1281/1281), done.
    remote: Compressing objects: 100% (812/812), done.
    remote: Total 3324 (delta 794), reused 841 (delta 461), pack-reused 2043
    Receiving objects: 100% (3324/3324), 23.05 MiB | 21.87 MiB/s, done.
    Resolving deltas: 100% (2067/2067), done.
    Looking in indexes: https://test.pypi.org/simple/, https://pypi.org/simple/
    Collecting sphinxcontrib-bibtex>=2.3.0
      Downloading sphinxcontrib_bibtex-2.4.1-py3-none-any.whl (38 kB)
    Collecting multipledispatch==0.6.0
      Downloading multipledispatch-0.6.0-py3-none-any.whl (11 kB)
    Collecting scikit-learn==0.23.0
      Downloading scikit_learn-0.23.0-cp37-cp37m-manylinux1_x86_64.whl (7.3 MB)
         |████████████████████████████████| 7.3 MB 12.8 MB/s
    Requirement already satisfied: numpy>=1.14.2 in /usr/local/lib/python3.7/dist-packages (from -r distil/requirements/requirements.txt (line 4)) (1.19.5)
    Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from -r distil/requirements/requirements.txt (line 5)) (1.4.1)
    Requirement already satisfied: torch>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from -r distil/requirements/requirements.txt (line 6)) (1.10.0+cu111)
    Requirement already satisfied: tqdm>=4.24.0 in /usr/local/lib/python3.7/dist-packages (from -r distil/requirements/requirements.txt (line 7)) (4.62.3)
    Requirement already satisfied: numba>=0.43.0 in /usr/local/lib/python3.7/dist-packages (from -r distil/requirements/requirements.txt (line 8)) (0.51.2)
    Collecting submodlib
      Downloading https://test-files.pythonhosted.org/packages/55/62/88e02a0e170498f38f7b9ce22b3e0a6a3cf9c82a33d3553da693c5c52872/submodlib-1.1.5.tar.gz (83 kB)
         |████████████████████████████████| 83 kB 1.1 MB/s
      Installing build dependencies ... done
      Getting requirements to build wheel ... done
        Preparing wheel metadata ... done
    Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from -r distil/requirements/requirements.txt (line 12)) (1.1.5)
    Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (from -r distil/requirements/requirements.txt (line 13)) (0.11.1+cu111)
    Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from multipledispatch==0.6.0->-r distil/requirements/requirements.txt (line 2)) (1.15.0)
    Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.23.0->-r distil/requirements/requirements.txt (line 3)) (1.1.0)
    Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.23.0->-r distil/requirements/requirements.txt (line 3)) (3.0.0)
    Collecting Sphinx>=2.1
      Downloading Sphinx-4.3.1-py3-none-any.whl (3.1 MB)
         |████████████████████████████████| 3.1 MB 55.1 MB/s
    Collecting pybtex-docutils>=1.0.0
      Downloading pybtex_docutils-1.0.1-py3-none-any.whl (4.8 kB)
    Requirement already satisfied: docutils>=0.8 in /usr/local/lib/python3.7/dist-packages (from sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (0.17.1)
    Collecting pybtex>=0.20
      Downloading pybtex-0.24.0-py2.py3-none-any.whl (561 kB)
         |████████████████████████████████| 561 kB 52.6 MB/s
    Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.4.0->-r distil/requirements/requirements.txt (line 6)) (3.10.0.2)
    Requirement already satisfied: llvmlite<0.35,>=0.34.0.dev0 in /usr/local/lib/python3.7/dist-packages (from numba>=0.43.0->-r distil/requirements/requirements.txt (line 8)) (0.34.0)
    Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from numba>=0.43.0->-r distil/requirements/requirements.txt (line 8)) (57.4.0)
    Requirement already satisfied: PyYAML>=3.01 in /usr/local/lib/python3.7/dist-packages (from pybtex>=0.20->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (3.13)
    Collecting latexcodec>=1.0.4
      Downloading latexcodec-2.0.1-py2.py3-none-any.whl (18 kB)
    Collecting sphinxcontrib-htmlhelp>=2.0.0
      Downloading sphinxcontrib_htmlhelp-2.0.0-py2.py3-none-any.whl (100 kB)
         |████████████████████████████████| 100 kB 12.9 MB/s
    Collecting sphinxcontrib-applehelp
      Downloading sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl (121 kB)
         |████████████████████████████████| 121 kB 55.1 MB/s
    Requirement already satisfied: sphinxcontrib-serializinghtml>=1.1.5 in /usr/local/lib/python3.7/dist-packages (from Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (1.1.5)
    Collecting sphinxcontrib-jsmath
      Downloading sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl (5.1 kB)
    Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.7/dist-packages (from Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (2.11.3)
    Collecting sphinxcontrib-devhelp
      Downloading sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl (84 kB)
         |████████████████████████████████| 84 kB 4.3 MB/s
    Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (2.2.0)
    Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (1.3.0)
    Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (21.3)
    Collecting sphinxcontrib-qthelp
      Downloading sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl (90 kB)
         |████████████████████████████████| 90 kB 11.3 MB/s
    Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (2.6.1)
    Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (0.7.12)
    Requirement already satisfied: babel>=1.3 in /usr/local/lib/python3.7/dist-packages (from Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (2.9.1)
    Requirement already satisfied: requests>=2.5.0 in /usr/local/lib/python3.7/dist-packages (from Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (2.23.0)
    Requirement already satisfied: pytz>=2015.7 in /usr/local/lib/python3.7/dist-packages (from babel>=1.3->Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (2018.9)
    Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.3->Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (2.0.1)
    Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.5.0->Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (1.24.3)
    Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.5.0->Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (2021.10.8)
    Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.5.0->Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (3.0.4)
    Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.5.0->Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (2.10)
    Requirement already satisfied: sklearn in /usr/local/lib/python3.7/dist-packages (from submodlib->-r distil/requirements/requirements.txt (line 11)) (0.0)
    Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->-r distil/requirements/requirements.txt (line 12)) (2.8.2)
    Requirement already satisfied: pillow!=8.3.0,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision->-r distil/requirements/requirements.txt (line 13)) (7.1.2)
    Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->Sphinx>=2.1->sphinxcontrib-bibtex>=2.3.0->-r distil/requirements/requirements.txt (line 1)) (3.0.6)
    Building wheels for collected packages: submodlib
      Building wheel for submodlib (PEP 517) ... done
      Created wheel for submodlib: filename=submodlib-1.1.5-cp37-cp37m-linux_x86_64.whl size=512655 sha256=5f42dc2a440923593a5ac15c93121875e37f2103e924b5f2e7d3836a671a3b77
      Stored in directory: /root/.cache/pip/wheels/17/3c/03/48ce7dd03798c0564b61e61020e733443aed88e115518442b4
    Successfully built submodlib
    Installing collected packages: latexcodec, sphinxcontrib-qthelp, sphinxcontrib-jsmath, sphinxcontrib-htmlhelp, sphinxcontrib-devhelp, sphinxcontrib-applehelp, scikit-learn, pybtex, Sphinx, pybtex-docutils, submodlib, sphinxcontrib-bibtex, multipledispatch
      Attempting uninstall: scikit-learn
        Found existing installation: scikit-learn 1.0.1
        Uninstalling scikit-learn-1.0.1:
          Successfully uninstalled scikit-learn-1.0.1
      Attempting uninstall: Sphinx
        Found existing installation: Sphinx 1.8.6
        Uninstalling Sphinx-1.8.6:
          Successfully uninstalled Sphinx-1.8.6
    ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
    imbalanced-learn 0.8.1 requires scikit-learn>=0.24, but you have scikit-learn 0.23.0 which is incompatible.
    datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.
    Successfully installed Sphinx-4.3.1 latexcodec-2.0.1 multipledispatch-0.6.0 pybtex-0.24.0 pybtex-docutils-1.0.1 scikit-learn-0.23.0 sphinxcontrib-applehelp-1.0.2 sphinxcontrib-bibtex-2.4.1 sphinxcontrib-devhelp-1.0.2 sphinxcontrib-htmlhelp-2.0.0 sphinxcontrib-jsmath-1.0.1 sphinxcontrib-qthelp-1.0.3 submodlib-1.1.5

## Preparing CIFAR-10 with Rare Classes

The CIFAR10 dataset contains 60,000 32x32 color images in 10 different classes.The 10 different classes represent airplanes, cars, birds, cats, deer, dogs, frogs, horses, ships, and trucks. There are 6,000 images of each class. The training set contains 50,000 images, and the test set contains 10,000 images. Here, we do a simple setup of the CIFAR10 dataset that we will use in this example. More importantly, we define a split on CIFAR10's training set into an initial labeled seed set and an unlabeled set. We also impose an artificial imbalance among the classes to simulate a rare-class scenario.

```python
# Define the name of the dataset and the path that PyTorch should use when downloading the data
data_set_name = 'CIFAR10'
download_path = '.'

# Define the number of classes in CIFAR10
nclasses = 10

# Define transforms on the dataset splits of CIFAR10. Here, we use random crops and horizontal flips for training augmentations.
# Both the train and test sets are converted to PyTorch tensors and are normalized around the mean/std of CIFAR-10.
cifar_training_transform = transforms.Compose([transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))])
cifar_test_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))])

# Get the dataset objects from PyTorch. Here, CIFAR10 is downloaded, and the transform is applied when points
# are retrieved.
cifar10_full_train = cifar.CIFAR10(download_path, train=True, download=True, transform=cifar_training_transform)
cifar10_test = cifar.CIFAR10(download_path, train=False, download=True, transform=cifar_test_transform)

# Get the dimension of the images. Here, we simply take the very first image of CIFAR10
# and query its dimension.
dim = np.shape(cifar10_full_train[0][0])

# We now define a train-unlabeled split for the sake of the experiment. Here, we specify the ratio of common classes to rare classes
# for both the initial training seed set and the unlabeled set. We use a ratio of 10:1.
num_examples_per_common_class_seed = 400
num_examples_per_rare_class_seed = 40

num_examples_per_common_class_unlabeled = 3000
num_examples_per_rare_class_unlabeled = 300

# We create the imbalance on classes 5,6,7,8,9.
rare_classes = [5,6,7,8,9]

# Create the imbalance by choosing which indices of the full training set to assign to the initial labeled seed set and the unlabeled set
train_idx = []
unlabeled_idx = []

for class_idx in range(nclasses):

    # Retrieve all the indices of the elements in CIFAR-10 whose label matches class_idx
    full_idx_class = torch.where(torch.Tensor(cifar10_full_train.targets) == class_idx)[0].cpu().numpy()

    # Determine how many points to add for this class depending on the rarity of the class
    if class_idx in rare_classes:
        class_num_training_examples_to_add = num_examples_per_rare_class_seed
        class_num_unlabeled_examples_to_add = num_examples_per_rare_class_unlabeled
    else:
        class_num_training_examples_to_add = num_examples_per_common_class_seed
        class_num_unlabeled_examples_to_add = num_examples_per_common_class_unlabeled

    # Choose randomly a subset of these indices. These will be added to the initial training seed set.
    class_train_idx = np.random.choice(full_idx_class, size=class_num_training_examples_to_add, replace=False)

    # Choose randomly a subset of the remaining indices. These will be added to the unlabeled set.
    remaining_class_idx = np.array(list(set(full_idx_class) - set(class_train_idx)))
    class_unlabeled_idx = np.random.choice(remaining_class_idx, size=class_num_unlabeled_examples_to_add, replace=False)

    # Add the chosen indices to the growing subset lists
    train_idx.extend(class_train_idx)
    unlabeled_idx.extend(class_unlabeled_idx)

# Create the train and unlabeled subsets based on the index lists above. While the unlabeled set constructed here technically has labels, they
# are only used when querying for labels. Hence, they only exist here for the sake of experimental design.
cifar10_train = Subset(cifar10_full_train, train_idx)
cifar10_unlabeled = Subset(cifar10_full_train, unlabeled_idx)
```

    Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz

      0%|          | 0/170498071 [00:00<?, ?it/s]

    Extracting ./cifar-10-python.tar.gz to .
    Files already downloaded and verified

## Preparing the Model

Here, we use DISTIL's provided implementation of the [ResNet-18](https://arxiv.org/abs/1512.03385) architecture. We also create a model directory to store trained models in this example.

```python
net = ResNet18()
```

# Initial Round

## Training

Now that we have prepared the training data and the model, we can begin training an initial model. We use DISTIL's provided [training loop](https://github.com/decile-team/distil/blob/main/distil/utils/train_helper.py) to do training.

```python
# Define the training arguments to use.
args = {'n_epoch':      300,    # Stop training after 300 epochs.
        'lr':           0.01,   # Use a learning rate of 0.01
        'batch_size':   20,     # Update the parameters using training batches of size 20
        'max_accuracy': 0.99,   # Stop training once the training accuracy has exceeded 0.99
        'optimizer':    'sgd',  # Use the stochastic gradient descent optimizer
        'device':       "cuda" if torch.cuda.is_available() else "cpu"  # Use a GPU if one is available
        }

# Create the training loop using our training dataset, provided model, and training arguments.
# Train an initial model.
dt = data_train(cifar10_train, net, args)
trained_model = dt.train()
```

    Training..
    Epoch: 105 Training accuracy: 0.991

## Evaluation

How does our initial model do on the test set? Furthermore, how does our initial model do on the *rare classes* of the test set? Luckily, the training loop provided by DISTIL also provides a way to measure the accuracy of the model on a given dataset. We measure both accuracies here.

```python
# Get the full test accuracy
full_test_accuracy = dt.get_acc_on_set(cifar10_test)
print(F"Full Test Accuracy: {full_test_accuracy}")

# Get the per-class test accuracies
rare_indices = []
for class_idx in range(nclasses):

    # Get the indices of the test set corresponding to this class
    test_rare_class_subset_idx = torch.where(torch.Tensor(cifar10_test.targets) == class_idx)[0].cpu().numpy()

    if class_idx in rare_classes:
        rare_indices.extend(test_rare_class_subset_idx)

    # Get the accuracy on this class subset
    cifar10_test_class_subset = Subset(cifar10_test, test_rare_class_subset_idx)
    rare_class_test_accuracy = dt.get_acc_on_set(cifar10_test_class_subset)
    print(F"Class {class_idx} Test Accuracy: {rare_class_test_accuracy}")

# Get accuracy on all rare points
cifar10_test_rare_subset = Subset(cifar10_test, rare_indices)
rare_test_accuracy = dt.get_acc_on_set(cifar10_test_rare_subset)
print(F"Rare Test Accuracy: {rare_test_accuracy}")
```

    Full Test Accuracy: 0.5509
    Class 0 Test Accuracy: 0.839
    Class 1 Test Accuracy: 0.894
    Class 2 Test Accuracy: 0.689
    Class 3 Test Accuracy: 0.738
    Class 4 Test Accuracy: 0.735
    Class 5 Test Accuracy: 0.142
    Class 6 Test Accuracy: 0.286
    Class 7 Test Accuracy: 0.36
    Class 8 Test Accuracy: 0.35
    Class 9 Test Accuracy: 0.476
    Rare Test Accuracy: 0.3228

As we can see, the test performance on the rare classes is awful. Can we rectify this issue by adding rare-class examples?

# Mining Rare Classes

## Preparing a Query Set

In this example, we know that there are rare classes in the unlabeled dataset that can help us improve the accuracy of the model. How do we select these points if we do not know *where* they are in the unlabeled dataset? Here, we use [SIMILAR](https://arxiv.org/abs/2107.00717) to select rare-class examples from the unlabeled set. SIMILAR requires access to a query set of points that it uses to choose similar unlabeled points. Hence, we must first prepare this query set.

Where do we get the query set? Luckily, we already have a couple points in our training dataset to choose.

```python
# Go over the training dataset, getting the indices of all the rare-class examples
rare_training_example_indices = []
for index, (_, label) in enumerate(cifar10_train):
    if label in rare_classes:
        rare_training_example_indices.append(index)

# Create a query set that contains only the rare-class examples of the training dataset
rare_class_query_set = Subset(cifar10_train, rare_training_example_indices)
```

## Using DISTIL's SIMILAR Implementation: SMI

Now that we have the query set, we are ready to use DISTIL's implementation of SIMILAR. In particular, we use the submodular mutual information [strategy](https://github.com/decile-team/distil/blob/main/distil/active_learning_strategies/smi.py) that is detailed in [SIMILAR](https://arxiv.org/abs/2107.00717). This will allow us to select a set of points to label within a specified budget $k$.

Specifically, the strategy attempts to maximize the [submodular mutual information](https://arxiv.org/abs/2006.15412) between a subset $\mathcal{A}$ of size no greater than $k$ of the unlabeled dataset $\mathcal{U}$ and the query set $\mathcal{Q}$:

\begin{align}
\text{argmax}_{\mathcal{A} \subseteq \mathcal{U}, |\mathcal{A}|\leq k} I_F(\mathcal{A};\mathcal{Q})
\end{align}

where $F$ is a submodular set function.

```python
# Define arguments for SMI
selection_strategy_args = {'device':        args['device'],       # Use the device used in training
                           'batch_size':    args['batch_size'],   # Use the batch size used in training
                           'smi_function':  'fl2mi',              # Use a facility location function, which captures representation information
                           'metric':        'cosine',             # Use cosine similarity when determining the likeness of two data points
                           'optimizer':     'LazyGreedy'          # When doing submodular maximization, use the lazy greedy optimizer
                          }

# Create the SMI selection strategy. Note: We remove the labels from the unlabeled portion of CIFAR-10 that we created earlier.
# In a practical application, one would not have these labels a priori.
selection_strategy = SMI(cifar10_train, LabeledToUnlabeledDataset(cifar10_unlabeled), rare_class_query_set, trained_model, nclasses, selection_strategy_args)

# Disable the augmentations used in the training dataset. Since all augmentations come from the cifar10_full_train object, we set its transform to the
# transform used by the test set.
cifar10_full_train.transform = cifar_test_transform

# Do the selection, which will return the indices of the selected points with respect to the unlabeled dataset.
budget = 750
selected_idx = selection_strategy.select(budget)

# Re-enable augmentations
cifar10_full_train.transform = cifar_training_transform
```

## Labeling the Points

Now that we know which points should be labeled, we can present them to human labelers for annotation. Here, we can do so automatically since we already know their labels for the sake of the example.

```python
# Form a labeled subset of the unlabeled dataset. Again, we already have the labels,
# so we simply take a subset. Note, however, that the selection was done without the
# use of the labels and that we would normally not have these labels. Hence, the
# following statement would usually require human effort to complete.
smi_human_labeled_dataset = Subset(cifar10_unlabeled, selected_idx)
```

## Characterizing the Selection

Now that we have selected and labeled these new points, we can add them to the training dataset and retrain our model. Before that, how many points did we select that actually were rare-class points?

```python
# Go over the newly labeled dataset, tallying the number of points seen in each class.
smi_class_counts = [0 for x in range(nclasses)]
for _, label in smi_human_labeled_dataset:
    smi_class_counts[label] += 1

# Print each class count
for class_idx, class_count in enumerate(smi_class_counts):
    print(F"Class {class_idx} count: {class_count}")

# Print total rare count
total_rare_count = sum([smi_class_counts[i] for i in rare_classes])
print(F"Total Rare Count: {total_rare_count}")
```

    Class 0 count: 47
    Class 1 count: 40
    Class 2 count: 43
    Class 3 count: 118
    Class 4 count: 81
    Class 5 count: 45
    Class 6 count: 65
    Class 7 count: 108
    Class 8 count: 74
    Class 9 count: 129
    Total Rare Count: 421

We were able to get a good number of rare-class points. For comparison sake, how many rare points would we get using some of DISTIL's other strategies?

**BADGE**

```python
# Repeat the previous steps
selection_strategy = BADGE(cifar10_train, LabeledToUnlabeledDataset(cifar10_unlabeled), trained_model, nclasses, selection_strategy_args)

cifar10_full_train.transform = cifar_test_transform
budget = 750
selected_idx = selection_strategy.select(budget)
cifar10_full_train.transform = cifar_training_transform

badge_human_labeled_dataset = Subset(cifar10_unlabeled, selected_idx)

badge_class_counts = [0 for x in range(nclasses)]
for _, label in badge_human_labeled_dataset:
    badge_class_counts[label] += 1

for class_idx, class_count in enumerate(badge_class_counts):
    print(F"Class {class_idx} count: {class_count}")

total_rare_count = sum([badge_class_counts[i] for i in rare_classes])
print(F"Total Rare Count: {total_rare_count}")
```

    Class 0 count: 114
    Class 1 count: 92
    Class 2 count: 139
    Class 3 count: 158
    Class 4 count: 125
    Class 5 count: 16
    Class 6 count: 29
    Class 7 count: 28
    Class 8 count: 19
    Class 9 count: 30
    Total Rare Count: 122

**Random**

```python
# Repeat the previous steps
selection_strategy = RandomSampling(cifar10_train, LabeledToUnlabeledDataset(cifar10_unlabeled), trained_model, nclasses, selection_strategy_args)

cifar10_full_train.transform = cifar_test_transform
budget = 750
selected_idx = selection_strategy.select(budget)
cifar10_full_train.transform = cifar_training_transform

random_human_labeled_dataset = Subset(cifar10_unlabeled, selected_idx)

random_class_counts = [0 for x in range(nclasses)]
for _, label in random_human_labeled_dataset:
    random_class_counts[label] += 1

for class_idx, class_count in enumerate(random_class_counts):
    print(F"Class {class_idx} count: {class_count}")

total_rare_count = sum([random_class_counts[i] for i in rare_classes])
print(F"Total Rare Count: {total_rare_count}")
```

    Class 0 count: 146
    Class 1 count: 128
    Class 2 count: 124
    Class 3 count: 142
    Class 4 count: 143
    Class 5 count: 16
    Class 6 count: 9
    Class 7 count: 14
    Class 8 count: 11
    Class 9 count: 17
    Total Rare Count: 67

**Entropy**

```python
# Repeat the previous steps
selection_strategy = EntropySampling(cifar10_train, LabeledToUnlabeledDataset(cifar10_unlabeled), trained_model, nclasses, selection_strategy_args)

cifar10_full_train.transform = cifar_test_transform
budget = 750
selected_idx = selection_strategy.select(budget)
cifar10_full_train.transform = cifar_training_transform

entropy_human_labeled_dataset = Subset(cifar10_unlabeled, selected_idx)

entropy_class_counts = [0 for x in range(nclasses)]
for _, label in entropy_human_labeled_dataset:
    entropy_class_counts[label] += 1

for class_idx, class_count in enumerate(entropy_class_counts):
    print(F"Class {class_idx} count: {class_count}")

total_rare_count = sum([entropy_class_counts[i] for i in rare_classes])
print(F"Total Rare Count: {total_rare_count}")
```

    Class 0 count: 96
    Class 1 count: 54
    Class 2 count: 126
    Class 3 count: 168
    Class 4 count: 172
    Class 5 count: 22
    Class 6 count: 34
    Class 7 count: 34
    Class 8 count: 22
    Class 9 count: 22
    Total Rare Count: 134

Hence, we can see that SMI does comparatively much better at selecting rare instances.

# Improving Performance

## Re-Training

Let us re-train our model using the newly selected points.

```python
# Create a new training dataset by concatenating what we have with the newly labeled points
new_training_dataset = ConcatDataset([cifar10_train, smi_human_labeled_dataset])
new_dt = data_train(new_training_dataset, copy.deepcopy(net), args)
new_trained_model = new_dt.train()
```

    Training..
    Epoch: 102 Training accuracy: 0.992

## Evaluation

Now, let us see the accuracy improvement.

```python
# Get the full test accuracy
full_test_accuracy_before = dt.get_acc_on_set(cifar10_test)
full_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test)

print(F"Full Test Accuracy Improvement: {full_test_accuracy_before} to {full_test_accuracy_after}")

# Get the per-class test accuracies
rare_indices = []
for class_idx in range(nclasses):

    # Get the indices of the test set corresponding to this class
    test_rare_class_subset_idx = torch.where(torch.Tensor(cifar10_test.targets) == class_idx)[0].cpu().numpy()

    if class_idx in rare_classes:
        rare_indices.extend(test_rare_class_subset_idx)

    # Get the accuracy on this class subset
    cifar10_test_class_subset = Subset(cifar10_test, test_rare_class_subset_idx)

    rare_class_test_accuracy_before = dt.get_acc_on_set(cifar10_test_class_subset)
    rare_class_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test_class_subset)
    print(F"Class {class_idx} Test Accuracy: {rare_class_test_accuracy_before} to {rare_class_test_accuracy_after}")

# Get accuracy on all rare points
cifar10_test_rare_subset = Subset(cifar10_test, rare_indices)
rare_test_accuracy_before = dt.get_acc_on_set(cifar10_test_rare_subset)
rare_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test_rare_subset)
print(F"Rare Test Accuracy: {rare_test_accuracy_before} to {rare_test_accuracy_after}")
```

    Full Test Accuracy Improvement: 0.5509 to 0.6512
    Class 0 Test Accuracy: 0.839 to 0.843
    Class 1 Test Accuracy: 0.894 to 0.918
    Class 2 Test Accuracy: 0.689 to 0.649
    Class 3 Test Accuracy: 0.738 to 0.769
    Class 4 Test Accuracy: 0.735 to 0.812
    Class 5 Test Accuracy: 0.142 to 0.283
    Class 6 Test Accuracy: 0.286 to 0.441
    Class 7 Test Accuracy: 0.36 to 0.537
    Class 8 Test Accuracy: 0.35 to 0.621
    Class 9 Test Accuracy: 0.476 to 0.639
    Rare Test Accuracy: 0.3228 to 0.5042

## Comparison

What would the accuracy improvement look like if we had used the other methods?

**BADGE**

```python
# Repeat the process
new_training_dataset = ConcatDataset([cifar10_train, badge_human_labeled_dataset])
new_dt = data_train(new_training_dataset, copy.deepcopy(net), args)
new_trained_model = new_dt.train()

full_test_accuracy_before = dt.get_acc_on_set(cifar10_test)
full_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test)

print(F"Full Test Accuracy Improvement: {full_test_accuracy_before} to {full_test_accuracy_after}")

rare_indices = []
for class_idx in range(nclasses):

    test_rare_class_subset_idx = torch.where(torch.Tensor(cifar10_test.targets) == class_idx)[0].cpu().numpy()

    if class_idx in rare_classes:
        rare_indices.extend(test_rare_class_subset_idx)

    cifar10_test_class_subset = Subset(cifar10_test, test_rare_class_subset_idx)

    rare_class_test_accuracy_before = dt.get_acc_on_set(cifar10_test_class_subset)
    rare_class_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test_class_subset)
    print(F"Class {class_idx} Test Accuracy: {rare_class_test_accuracy_before} to {rare_class_test_accuracy_after}")

cifar10_test_rare_subset = Subset(cifar10_test, rare_indices)
rare_test_accuracy_before = dt.get_acc_on_set(cifar10_test_rare_subset)
rare_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test_rare_subset)
print(F"Rare Test Accuracy: {rare_test_accuracy_before} to {rare_test_accuracy_after}")
```

    Training..
    Epoch: 122 Training accuracy: 0.994
    Full Test Accuracy Improvement: 0.5509 to 0.623
    Class 0 Test Accuracy: 0.839 to 0.883
    Class 1 Test Accuracy: 0.894 to 0.941
    Class 2 Test Accuracy: 0.689 to 0.725
    Class 3 Test Accuracy: 0.738 to 0.748
    Class 4 Test Accuracy: 0.735 to 0.797
    Class 5 Test Accuracy: 0.142 to 0.276
    Class 6 Test Accuracy: 0.286 to 0.312
    Class 7 Test Accuracy: 0.36 to 0.492
    Class 8 Test Accuracy: 0.35 to 0.441
    Class 9 Test Accuracy: 0.476 to 0.615
    Rare Test Accuracy: 0.3228 to 0.4272

**Random**

```python
# Repeat the process
new_training_dataset = ConcatDataset([cifar10_train, random_human_labeled_dataset])
new_dt = data_train(new_training_dataset, copy.deepcopy(net), args)
new_trained_model = new_dt.train()

full_test_accuracy_before = dt.get_acc_on_set(cifar10_test)
full_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test)

print(F"Full Test Accuracy Improvement: {full_test_accuracy_before} to {full_test_accuracy_after}")

rare_indices = []
for class_idx in range(nclasses):

    test_rare_class_subset_idx = torch.where(torch.Tensor(cifar10_test.targets) == class_idx)[0].cpu().numpy()

    if class_idx in rare_classes:
        rare_indices.extend(test_rare_class_subset_idx)

    cifar10_test_class_subset = Subset(cifar10_test, test_rare_class_subset_idx)

    rare_class_test_accuracy_before = dt.get_acc_on_set(cifar10_test_class_subset)
    rare_class_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test_class_subset)
    print(F"Class {class_idx} Test Accuracy: {rare_class_test_accuracy_before} to {rare_class_test_accuracy_after}")

cifar10_test_rare_subset = Subset(cifar10_test, rare_indices)
rare_test_accuracy_before = dt.get_acc_on_set(cifar10_test_rare_subset)
rare_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test_rare_subset)
print(F"Rare Test Accuracy: {rare_test_accuracy_before} to {rare_test_accuracy_after}")
```

    Training..
    Epoch: 118 Training accuracy: 0.991
    Full Test Accuracy Improvement: 0.5509 to 0.5915
    Class 0 Test Accuracy: 0.839 to 0.771
    Class 1 Test Accuracy: 0.894 to 0.932
    Class 2 Test Accuracy: 0.689 to 0.758
    Class 3 Test Accuracy: 0.738 to 0.612
    Class 4 Test Accuracy: 0.735 to 0.806
    Class 5 Test Accuracy: 0.142 to 0.232
    Class 6 Test Accuracy: 0.286 to 0.425
    Class 7 Test Accuracy: 0.36 to 0.341
    Class 8 Test Accuracy: 0.35 to 0.499
    Class 9 Test Accuracy: 0.476 to 0.539
    Rare Test Accuracy: 0.3228 to 0.4072

**Entropy**

```python
# Repeat the process
new_training_dataset = ConcatDataset([cifar10_train, entropy_human_labeled_dataset])
new_dt = data_train(new_training_dataset, copy.deepcopy(net), args)
new_trained_model = new_dt.train()

full_test_accuracy_before = dt.get_acc_on_set(cifar10_test)
full_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test)

print(F"Full Test Accuracy Improvement: {full_test_accuracy_before} to {full_test_accuracy_after}")

rare_indices = []
for class_idx in range(nclasses):

    test_rare_class_subset_idx = torch.where(torch.Tensor(cifar10_test.targets) == class_idx)[0].cpu().numpy()

    if class_idx in rare_classes:
        rare_indices.extend(test_rare_class_subset_idx)

    cifar10_test_class_subset = Subset(cifar10_test, test_rare_class_subset_idx)

    rare_class_test_accuracy_before = dt.get_acc_on_set(cifar10_test_class_subset)
    rare_class_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test_class_subset)
    print(F"Class {class_idx} Test Accuracy: {rare_class_test_accuracy_before} to {rare_class_test_accuracy_after}")

cifar10_test_rare_subset = Subset(cifar10_test, rare_indices)
rare_test_accuracy_before = dt.get_acc_on_set(cifar10_test_rare_subset)
rare_test_accuracy_after = new_dt.get_acc_on_set(cifar10_test_rare_subset)
print(F"Rare Test Accuracy: {rare_test_accuracy_before} to {rare_test_accuracy_after}")
```

    Training..
    Epoch: 116 Training accuracy: 0.993
    Full Test Accuracy Improvement: 0.5509 to 0.6483
    Class 0 Test Accuracy: 0.839 to 0.879
    Class 1 Test Accuracy: 0.894 to 0.906
    Class 2 Test Accuracy: 0.689 to 0.699
    Class 3 Test Accuracy: 0.738 to 0.769
    Class 4 Test Accuracy: 0.735 to 0.829
    Class 5 Test Accuracy: 0.142 to 0.182
    Class 6 Test Accuracy: 0.286 to 0.456
    Class 7 Test Accuracy: 0.36 to 0.457
    Class 8 Test Accuracy: 0.35 to 0.623
    Class 9 Test Accuracy: 0.476 to 0.683
    Rare Test Accuracy: 0.3228 to 0.4802