{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "device(type='cpu')" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import gc\n", "import os\n", "import cv2\n", "import sys\n", "import json\n", "import time\n", "import timm\n", "import torch\n", "import tqdm\n", "import random\n", "import sklearn.metrics\n", "\n", "from PIL import Image\n", "from pathlib import Path\n", "from functools import partial\n", "from contextlib import contextmanager\n", "from scipy.special import softmax\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import torch.nn as nn\n", "\n", "from torch.optim import Adam, SGD\n", "from torch.optim.lr_scheduler import CosineAnnealingLR\n", "from torch.utils.data import DataLoader, Dataset\n", "from albumentations import Compose, Normalize, Resize\n", "from albumentations.pytorch import ToTensorV2\n", "from sklearn.preprocessing import LabelBinarizer\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\"\n", "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", "device" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting timm\n", " Downloading timm-1.0.20-py3-none-any.whl.metadata (61 kB)\n", "Requirement already satisfied: torch in c:\\users\\general\\anaconda3\\lib\\site-packages (from timm) (2.9.0)\n", "Requirement already satisfied: torchvision in c:\\users\\general\\anaconda3\\lib\\site-packages (from timm) (0.24.0)\n", "Requirement already satisfied: pyyaml in c:\\users\\general\\anaconda3\\lib\\site-packages (from timm) (6.0.2)\n", "Collecting huggingface_hub (from timm)\n", " Downloading huggingface_hub-0.35.3-py3-none-any.whl.metadata (14 kB)\n", "Collecting safetensors (from timm)\n", " Downloading safetensors-0.6.2-cp38-abi3-win_amd64.whl.metadata (4.1 kB)\n", "Requirement already satisfied: filelock in c:\\users\\general\\anaconda3\\lib\\site-packages (from huggingface_hub->timm) (3.17.0)\n", "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\general\\anaconda3\\lib\\site-packages (from huggingface_hub->timm) (2025.3.2)\n", "Requirement already satisfied: packaging>=20.9 in c:\\users\\general\\anaconda3\\lib\\site-packages (from huggingface_hub->timm) (24.2)\n", "Requirement already satisfied: requests in c:\\users\\general\\anaconda3\\lib\\site-packages (from huggingface_hub->timm) (2.32.3)\n", "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\general\\anaconda3\\lib\\site-packages (from huggingface_hub->timm) (4.67.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\general\\anaconda3\\lib\\site-packages (from huggingface_hub->timm) (4.12.2)\n", "Requirement already satisfied: colorama in c:\\users\\general\\anaconda3\\lib\\site-packages (from tqdm>=4.42.1->huggingface_hub->timm) (0.4.6)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\general\\anaconda3\\lib\\site-packages (from requests->huggingface_hub->timm) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\general\\anaconda3\\lib\\site-packages (from requests->huggingface_hub->timm) (3.7)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\general\\anaconda3\\lib\\site-packages (from requests->huggingface_hub->timm) (2.3.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\general\\anaconda3\\lib\\site-packages (from requests->huggingface_hub->timm) (2025.4.26)\n", "Requirement already satisfied: sympy>=1.13.3 in c:\\users\\general\\anaconda3\\lib\\site-packages (from torch->timm) (1.13.3)\n", "Requirement already satisfied: networkx>=2.5.1 in c:\\users\\general\\anaconda3\\lib\\site-packages (from torch->timm) (3.4.2)\n", "Requirement already satisfied: jinja2 in c:\\users\\general\\anaconda3\\lib\\site-packages (from torch->timm) (3.1.6)\n", "Requirement already satisfied: setuptools in c:\\users\\general\\anaconda3\\lib\\site-packages (from torch->timm) (72.1.0)\n", "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\general\\anaconda3\\lib\\site-packages (from sympy>=1.13.3->torch->timm) (1.3.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\general\\anaconda3\\lib\\site-packages (from jinja2->torch->timm) (3.0.2)\n", "Requirement already satisfied: numpy in c:\\users\\general\\anaconda3\\lib\\site-packages (from torchvision->timm) (2.1.3)\n", "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in c:\\users\\general\\anaconda3\\lib\\site-packages (from torchvision->timm) (11.1.0)\n", "Downloading timm-1.0.20-py3-none-any.whl (2.5 MB)\n", " ---------------------------------------- 0.0/2.5 MB ? eta -:--:--\n", " ---------------------------------------- 0.0/2.5 MB ? eta -:--:--\n", " ---------------------------------------- 0.0/2.5 MB ? eta -:--:--\n", " ---------------------------------------- 0.0/2.5 MB ? eta -:--:--\n", " ---- ----------------------------------- 0.3/2.5 MB ? eta -:--:--\n", " -------- ------------------------------- 0.5/2.5 MB 980.0 kB/s eta 0:00:03\n", " -------- ------------------------------- 0.5/2.5 MB 980.0 kB/s eta 0:00:03\n", " -------- ------------------------------- 0.5/2.5 MB 980.0 kB/s eta 0:00:03\n", " ------------ --------------------------- 0.8/2.5 MB 542.1 kB/s eta 0:00:04\n", " ------------ --------------------------- 0.8/2.5 MB 542.1 kB/s eta 0:00:04\n", " ---------------- ----------------------- 1.0/2.5 MB 566.9 kB/s eta 0:00:03\n", " ---------------- ----------------------- 1.0/2.5 MB 566.9 kB/s eta 0:00:03\n", " -------------------- ------------------- 1.3/2.5 MB 630.6 kB/s eta 0:00:02\n", " -------------------- ------------------- 1.3/2.5 MB 630.6 kB/s eta 0:00:02\n", " -------------------- ------------------- 1.3/2.5 MB 630.6 kB/s eta 0:00:02\n", " ----------------------------- ---------- 1.8/2.5 MB 641.9 kB/s eta 0:00:02\n", " ----------------------------- ---------- 1.8/2.5 MB 641.9 kB/s eta 0:00:02\n", " ----------------------------- ---------- 1.8/2.5 MB 641.9 kB/s eta 0:00:02\n", " --------------------------------- ------ 2.1/2.5 MB 634.3 kB/s eta 0:00:01\n", " --------------------------------- ------ 2.1/2.5 MB 634.3 kB/s eta 0:00:01\n", " ------------------------------------- -- 2.4/2.5 MB 613.0 kB/s eta 0:00:01\n", " ------------------------------------- -- 2.4/2.5 MB 613.0 kB/s eta 0:00:01\n", " ---------------------------------------- 2.5/2.5 MB 587.5 kB/s eta 0:00:00\n", "Downloading huggingface_hub-0.35.3-py3-none-any.whl (564 kB)\n", " ---------------------------------------- 0.0/564.3 kB ? eta -:--:--\n", " ---------------------------------------- 0.0/564.3 kB ? eta -:--:--\n", " ---------------------------------------- 0.0/564.3 kB ? eta -:--:--\n", " ---------------------------------------- 0.0/564.3 kB ? eta -:--:--\n", " ------------------ --------------------- 262.1/564.3 kB ? eta -:--:--\n", " ---------------------------------------- 564.3/564.3 kB 1.1 MB/s eta 0:00:00\n", "Downloading safetensors-0.6.2-cp38-abi3-win_amd64.whl (320 kB)\n", "Installing collected packages: safetensors, huggingface_hub, timm\n", "\n", " ------------- -------------------------- 1/3 [huggingface_hub]\n", " ------------- -------------------------- 1/3 [huggingface_hub]\n", " ------------- -------------------------- 1/3 [huggingface_hub]\n", " ------------- -------------------------- 1/3 [huggingface_hub]\n", " ------------- -------------------------- 1/3 [huggingface_hub]\n", " -------------------------- ------------- 2/3 [timm]\n", " -------------------------- ------------- 2/3 [timm]\n", " -------------------------- ------------- 2/3 [timm]\n", " -------------------------- ------------- 2/3 [timm]\n", " -------------------------- ------------- 2/3 [timm]\n", " -------------------------- ------------- 2/3 [timm]\n", " -------------------------- ------------- 2/3 [timm]\n", " -------------------------- ------------- 2/3 [timm]\n", " -------------------------- ------------- 2/3 [timm]\n", " -------------------------- ------------- 2/3 [timm]\n", " ---------------------------------------- 3/3 [timm]\n", "\n", "Successfully installed huggingface_hub-0.35.3 safetensors-0.6.2 timm-1.0.20\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip install timm\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Loading Metadata" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "386006 772\n" ] } ], "source": [ "metadata = pd.read_csv(\"train_metadata.csv\")\n", "labels_species = metadata['binomial']\n", "lb_species = LabelBinarizer()\n", "lb_species.fit(np.asarray(labels_species))\n", "\n", "print(len(metadata), len(metadata['binomial'].unique()))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_metadata = pd.read_csv(\"../metadata/SnakeCLEF2021_TEST_METADATA-PRIVATE.csv\")\n", "test_metadata[\"image_path\"] = test_metadata['file_path'].apply(lambda x: \"/local/nahouby/Datasets/SnakeCLEF2021-test/test-inat-post-4.2/\" + x)\n", "test_metadata.fillna('unknown', inplace=True)\n", "\n", "test_metadata.head(5)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "country_relevance = pd.read_csv(\"../metadata/Species_Relevance.csv\", sep=';')\n", "country_relevance = country_relevance.rename(columns={'Unnamed: 0': 'binomial'})\n", "KO_species = set(country_relevance['binomial']) - set(metadata['binomial'])\n", "country_relevance = country_relevance[~country_relevance.binomial.isin(KO_species)]\n", "country_relevance = country_relevance.reset_index().drop(columns=['index'])\n", "country_relevance['class_id'] = country_relevance.apply(lambda row: np.where(lb_species.classes_ == row['binomial'])[0][0], axis=1)\n", "country_relevance = country_relevance.sort_values(by=['class_id'])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Loading Model" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def getModel(architecture_name, target_size, pretrained = False):\n", " net = timm.create_model(architecture_name, pretrained=pretrained)\n", " net_cfg = net.default_cfg\n", " last_layer = net_cfg['classifier']\n", " num_ftrs = getattr(net, last_layer).in_features\n", " setattr(net, last_layer, nn.Linear(num_ftrs, target_size))\n", " return net" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Done.\n" ] } ], "source": [ "# %%\n", "N_CLASSES = 772\n", "MODEL_NAME = 'vit_large_patch16_384'\n", "model = getModel(MODEL_NAME, N_CLASSES, pretrained=True)\n", "model_mean = list(model.default_cfg['mean'])\n", "model_std = list(model.default_cfg['std'])\n", "\n", "model.load_state_dict(torch.load('../../SnakeCLEF2021/CKPTS/SnakeCLEF2021-ViT_large_patch16-384-FT-FL-OCLR-20E.pth'))\n", "\n", "model.to(device)\n", "model.eval()\n", "print('Done.')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Load Data etc." ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def seed_torch(seed=777):\n", " random.seed(seed)\n", " os.environ['PYTHONHASHSEED'] = str(seed)\n", " np.random.seed(seed)\n", " torch.manual_seed(seed)\n", " torch.cuda.manual_seed(seed)\n", " torch.backends.cudnn.deterministic = True\n", "\n", " \n", "class CustomDataset(Dataset):\n", " def __init__(self, df, transform=None):\n", " self.df = df\n", " self.transform = transform\n", " \n", " def __len__(self):\n", " return len(self.df)\n", "\n", " def __getitem__(self, idx):\n", " file_path = self.df['image_path'].values[idx]\n", " continent = self.df['continent'].values[idx]\n", " country = self.df['country'].values[idx]\n", " class_id = self.df['class_id'].values[idx]\n", " image = cv2.imread(file_path)\n", " try:\n", " image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n", " except:\n", " print(file_path)\n", " \n", " if self.transform:\n", " augmented = self.transform(image=image)\n", " image = augmented['image']\n", " \n", " return image, file_path, class_id, country, continent\n", " \n", "SEED = 777\n", "seed_torch(SEED)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "HEIGHT = 384\n", "WIDTH = 384\n", "\n", "from albumentations import RandomGridShuffle, CenterCrop, HueSaturationValue, RandomCrop, HorizontalFlip, VerticalFlip, RandomBrightnessContrast, CenterCrop, PadIfNeeded, RandomResizedCrop, ShiftScaleRotate, Blur, JpegCompression, RandomShadow\n", "\n", "def get_transforms(*, data):\n", " assert data in ('train', 'test1', 'test2', 'test3')\n", "\n", " if data == 'train':\n", " return Compose([\n", " RandomResizedCrop(WIDTH, HEIGHT, scale=(0.7, 1.0)),\n", " HorizontalFlip(p=0.5),\n", " VerticalFlip(p=0.5),\n", " ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.25, rotate_limit=45, p=.75),\n", " JpegCompression(quality_lower=50, quality_upper=100),\n", " #RandomShadow(),\n", " Blur(blur_limit=2),\n", " RandomBrightnessContrast(p=0.3),\n", " HueSaturationValue(p=0.2),\n", " Normalize(\n", " mean=model_mean,\n", " std=model_std,\n", " ),\n", " ToTensorV2(),\n", " ])\n", "\n", " elif data == 'test1':\n", " return Compose([\n", " Resize(WIDTH, HEIGHT),\n", " Normalize(mean = model_mean, std = model_std),\n", " ToTensorV2(),\n", " ])\n", " elif data == 'test2':\n", " return Compose([\n", " PadIfNeeded(WIDTH, HEIGHT),\n", " Resize(int(WIDTH*1.2), int(HEIGHT*1.2)),\n", " CenterCrop(WIDTH, HEIGHT),\n", " Normalize(mean = model_mean, std = model_std),\n", " ToTensorV2(),\n", " ])\n", " elif data == 'test3':\n", " return Compose([\n", " PadIfNeeded(WIDTH, HEIGHT),\n", " Resize(int(WIDTH*1.5), int(HEIGHT*1.5)),\n", " CenterCrop(WIDTH, HEIGHT),\n", " Normalize(mean = model_mean, std = model_std),\n", " ToTensorV2(),\n", " ])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Testing" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "BATCH_SIZE = 16\n", "WORKERS = 8\n", "\n", "test_dataset_1 = CustomDataset(test_metadata, transform=get_transforms(data='test1'))\n", "test_loader_1 = DataLoader(test_dataset_1, batch_size=BATCH_SIZE, shuffle=False, num_workers=WORKERS)\n", "\n", "test_dataset_2 = CustomDataset(test_metadata, transform=get_transforms(data='test2'))\n", "test_loader_2 = DataLoader(test_dataset_2, batch_size=BATCH_SIZE, shuffle=False, num_workers=WORKERS)\n", "\n", "test_dataset_3 = CustomDataset(test_metadata, transform=get_transforms(data='test3'))\n", "test_loader_3 = DataLoader(test_dataset_3, batch_size=BATCH_SIZE, shuffle=False, num_workers=WORKERS)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Test with Augmentations v1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "preds = np.zeros((len(test_metadata)), dtype=np.int64)\n", "GT_lbls = []\n", "image_paths = []\n", "preds_raw = []\n", "countries = []\n", "continents = []\n", "\n", "for i, (images, paths, labels, counts, conts) in enumerate(tqdm.tqdm(test_loader_1, total=len(test_loader_1))):\n", "\n", " images = images.to(device)\n", " labels = labels.to(device)\n", " \n", " with torch.no_grad():\n", " y_preds = model(images)\n", " \n", " preds[i * BATCH_SIZE: (i+1) * BATCH_SIZE] = y_preds.argmax(1).to('cpu').numpy()\n", " GT_lbls.extend(labels.to('cpu').numpy())\n", " preds_raw.extend(y_preds.to('cpu').numpy())\n", " image_paths.extend(paths)\n", " countries.extend(counts)\n", " continents.extend(conts)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "test_metadata['logits_t1'] = preds_raw\n", "test_metadata['preds_t1'] = [np.argmax(p) for p in preds_raw]" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Test Augmentation 1: 88.77 94.13\n" ] } ], "source": [ "from sklearn.metrics import f1_score, accuracy_score, top_k_accuracy_score, classification_report\n", "\n", "vanilla_f1 = f1_score(test_metadata['class_id'], test_metadata['preds_t1'], average='macro')\n", "vanilla_accuracy = accuracy_score(test_metadata['class_id'], test_metadata['preds_t1'])\n", "\n", "print('Test Augmentation 1:', np.round(vanilla_f1 * 100, 2), np.round(vanilla_accuracy * 100, 2))" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "output = pd.DataFrame(zip(test_metadata.UUID, preds), columns =['UUID', 'prediction'])\n", "output.to_csv('vanilla.csv', index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Test with Augmentations v2" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "preds = np.zeros((len(test_metadata)))\n", "GT_lbls = []\n", "image_paths = []\n", "preds_raw = []\n", "countries = []\n", "continents = []\n", "\n", "for i, (images, paths, labels, counts, conts) in enumerate(tqdm.tqdm(test_loader_2, total=len(test_loader_2))):\n", "\n", " images = images.to(device)\n", " labels = labels.to(device)\n", " \n", " with torch.no_grad():\n", " y_preds = model(images)\n", " \n", " preds[i * BATCH_SIZE: (i+1) * BATCH_SIZE] = y_preds.argmax(1).to('cpu').numpy()\n", " GT_lbls.extend(labels.to('cpu').numpy())\n", " preds_raw.extend(y_preds.to('cpu').numpy())\n", " image_paths.extend(paths)\n", " countries.extend(counts)\n", " continents.extend(conts)" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "test_metadata['logits_t2'] = preds_raw\n", "test_metadata['preds_t2'] = [np.argmax(p) for p in preds_raw]" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Test Augmentation 2: 88.68 94.55\n" ] } ], "source": [ "from sklearn.metrics import f1_score, accuracy_score, top_k_accuracy_score, classification_report\n", "\n", "vanilla_f1 = f1_score(test_metadata['class_id'], test_metadata['preds_t2'], average='macro')\n", "vanilla_accuracy = accuracy_score(test_metadata['class_id'], test_metadata['preds_t2'])\n", "\n", "print('Test Augmentation 2:', np.round(vanilla_f1 * 100, 2), np.round(vanilla_accuracy * 100, 2))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Test with Augmentations v3" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "preds = np.zeros((len(test_metadata)))\n", "GT_lbls = []\n", "image_paths = []\n", "preds_raw = []\n", "countries = []\n", "continents = []\n", "\n", "for i, (images, paths, labels, counts, conts) in enumerate(tqdm.tqdm(test_loader_3, total=len(test_loader_3))):\n", "\n", " images = images.to(device)\n", " labels = labels.to(device)\n", " \n", " with torch.no_grad():\n", " y_preds = model(images)\n", " \n", " preds[i * BATCH_SIZE: (i+1) * BATCH_SIZE] = y_preds.argmax(1).to('cpu').numpy()\n", " GT_lbls.extend(labels.to('cpu').numpy())\n", " preds_raw.extend(y_preds.to('cpu').numpy())\n", " image_paths.extend(paths)\n", " countries.extend(counts)\n", " continents.extend(conts)" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [], "source": [ "test_metadata['logits_t3'] = preds_raw\n", "test_metadata['preds_t3'] = [np.argmax(p) for p in preds_raw]" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Test Augmentation 3: 88.58 93.92\n" ] } ], "source": [ "from sklearn.metrics import f1_score, accuracy_score, top_k_accuracy_score, classification_report\n", "\n", "vanilla_f1 = f1_score(test_metadata['class_id'], test_metadata['preds_t3'], average='macro')\n", "vanilla_accuracy = accuracy_score(test_metadata['class_id'], test_metadata['preds_t3'])\n", "\n", "print('Test Augmentation 3:', np.round(vanilla_f1 * 100, 2), np.round(vanilla_accuracy * 100, 2))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Prediction Mean" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_metadata['mean_softmax'] = 0\n", "\n", "for index, row in tqdm.tqdm(test_metadata.iterrows(), total=len(test_metadata)):\n", " max_index = np.argmax(sum((softmax(row.logits_t1), softmax(row.logits_t2), softmax(row.logits_t3))))\n", " test_metadata.at[index, 'mean_softmax'] = max_index" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Mean softmax: 89.13 95.15\n" ] } ], "source": [ "vanilla_f1 = f1_score(test_metadata['class_id'], test_metadata['mean_softmax'], average='macro')\n", "vanilla_accuracy = accuracy_score(test_metadata['class_id'], test_metadata['mean_softmax'])\n", "\n", "print('Mean softmax:', np.round(vanilla_f1 * 100, 2), np.round(vanilla_accuracy * 100, 2))" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [], "source": [ "output = pd.DataFrame(zip(test_metadata.UUID, test_metadata.mean_softmax), columns =['UUID', 'prediction'])\n", "output.to_csv('mean_softmax.csv', index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Metadata extraction" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Extracting Species distribution" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [], "source": [ "class_priors = np.ones(len(metadata['class_id'].unique()))\n", "for species in metadata['class_id'].unique():\n", " class_priors[species] = len(metadata[metadata['class_id'] == species])\n", "\n", "class_priors = class_priors / sum(class_priors)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class_2_genus = {}\n", "for _, row in tqdm.tqdm(metadata.iterrows(), total=len(metadata)):\n", " if row.class_id not in class_2_genus:\n", " class_2_genus[row.class_id] = row.genus\n", " \n", "class_2_family = {}\n", "for _, row in tqdm.tqdm(metadata.iterrows(), total=len(metadata)):\n", " if row.class_id not in class_2_family:\n", " class_2_family[row.class_id] = row.family" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Country Distribution" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "country_distributions = {}\n", "\n", "for _, observation in tqdm.tqdm(metadata.iterrows(), total=len(metadata)):\n", " country = str(observation.country)\n", " class_id = observation.class_id\n", " if country not in country_distributions: \n", " country_distributions[country] = np.ones(len(metadata['class_id'].unique()))\n", " else:\n", " country_distributions[country][class_id] += 1\n", "\n", "for key, value in country_distributions.items():\n", " country_distributions[key] = value / sum(value)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Continent Distribution" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "continent_distributions = {}\n", "\n", "for _, observation in tqdm.tqdm(metadata.iterrows(), total=len(metadata)):\n", " continent = str(observation.continent)\n", " class_id = observation.class_id\n", " if continent not in continent_distributions: \n", " continent_distributions[continent] = np.ones(len(metadata['class_id'].unique()))\n", " else:\n", " continent_distributions[continent][class_id] += 1\n", "\n", "for key, value in continent_distributions.items():\n", " continent_distributions[key] = value / sum(value)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Prior Weighting" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Binary filtration" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23673/23673 [00:04<00:00, 4931.83it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Binary Masking:\n", "F1: 92.24 Acc: 96.0\n", "F1 dif: 3.11 Acc dif: 0.84\n" ] } ], "source": [ "test_metadata['binary_filtration'] = 0\n", "\n", "for index, row in tqdm.tqdm(test_metadata.iterrows(), total=len(test_metadata)):\n", "\n", " country = row.country\n", " \n", " preds = sum((softmax(row.logits_t1), softmax(row.logits_t2), softmax(row.logits_t3))) / 3\n", "\n", " if country == 'Republic of Congo':\n", " country = 'republic of the congo'\n", "\n", " if country in ['unknown', 'West Bank', 'Macau S.A.R', 'US Naval Base Guantanamo Bay', 'United States Virgin Islands', 'Guam', 'Turks and Caicos Islands', 'Cyprus No Mans Area']:\n", " preds = preds\n", " else:\n", " preds = preds * np.array(country_relevance[country.lower()], dtype=np.int32)\n", "\n", " max_index = np.argmax(preds) \n", " \n", " test_metadata.at[index, 'binary_filtration'] = max_index\n", "\n", "f1 = f1_score(test_metadata['class_id'], test_metadata['binary_filtration'], average='macro')\n", "accuracy = accuracy_score(test_metadata['class_id'], test_metadata['binary_filtration'])\n", "print('Binary Masking:')\n", "print('F1:', np.round(f1 * 100, 2), 'Acc:', np.round(accuracy * 100, 2))\n", "print('F1 dif:', np.round((f1-vanilla_f1) * 100, 2), 'Acc dif:', np.round((accuracy-vanilla_accuracy) * 100, 2)) " ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [], "source": [ "output = pd.DataFrame(zip(test_metadata.UUID, test_metadata.binary_filtration), columns =['UUID', 'prediction'])\n", "output.to_csv('masking.csv', index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Country Weighting" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23673/23673 [00:06<00:00, 3553.41it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Binary Masking:\n", "F1: 92.27 Acc: 95.92\n", "F1 dif: 3.14 Acc dif: 0.77\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "test_metadata['country_weighting'] = 0\n", "\n", "\n", "for index, row in tqdm.tqdm(test_metadata.iterrows(), total=len(test_metadata)):\n", "\n", " country = row.country\n", " preds = sum((softmax(row.logits_t1), softmax(row.logits_t2), softmax(row.logits_t3))) / 3\n", " \n", " if country not in country_distributions:\n", " country = 'unknown'\n", " country_dist = country_distributions[country]\n", " \n", " p_countries = (preds * country_dist) / sum(preds * country_dist)\n", " prior_ratio = p_countries / class_priors\n", " max_index = np.argmax(prior_ratio * preds) \n", " \n", " test_metadata.at[index, 'country_weighting'] = max_index\n", " \n", "f1 = f1_score(test_metadata['class_id'], test_metadata['country_weighting'], average='macro')\n", "accuracy = accuracy_score(test_metadata['class_id'], test_metadata['country_weighting'])\n", "print('Binary Masking:')\n", "print('F1:', np.round(f1 * 100, 2), 'Acc:', np.round(accuracy * 100, 2))\n", "print('F1 dif:', np.round((f1-vanilla_f1) * 100, 2), 'Acc dif:', np.round((accuracy-vanilla_accuracy) * 100, 2)) " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "output = pd.DataFrame(zip(test_metadata.UUID, test_metadata.country_weighting), columns =['UUID', 'prediction'])\n", "output.to_csv('country_weighting.csv', index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Continent Weighting" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23673/23673 [00:06<00:00, 3609.67it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Binary Masking:\n", "F1: 91.16 Acc: 95.15\n", "F1 dif: 2.03 Acc dif: 0.0\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "test_metadata['continent_weighting'] = 0\n", "\n", "\n", "for index, row in tqdm.tqdm(test_metadata.iterrows(), total=len(test_metadata)):\n", "\n", " continent = row.continent\n", " preds = sum((softmax(row.logits_t1), softmax(row.logits_t2), softmax(row.logits_t3))) / 3\n", " \n", " continent_dist = continent_distributions[continent]\n", " \n", " p_continent = (preds * continent_dist) / sum(preds * continent_dist)\n", " prior_ratio = p_continent / class_priors\n", " max_index = np.argmax(prior_ratio * preds) \n", " \n", " test_metadata.at[index, 'continent_weighting'] = max_index\n", " \n", "f1 = f1_score(test_metadata['class_id'], test_metadata['continent_weighting'], average='macro')\n", "accuracy = accuracy_score(test_metadata['class_id'], test_metadata['continent_weighting'])\n", "print('Binary Masking:')\n", "print('F1:', np.round(f1 * 100, 2), 'Acc:', np.round(accuracy * 100, 2))\n", "print('F1 dif:', np.round((f1-vanilla_f1) * 100, 2), 'Acc dif:', np.round((accuracy-vanilla_accuracy) * 100, 2)) " ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [], "source": [ "output = pd.DataFrame(zip(test_metadata.UUID, test_metadata.continent_weighting), columns =['UUID', 'prediction'])\n", "output.to_csv('continent_weighting.csv', index=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Continent Weighting + Binary Masking" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 23673/23673 [00:07<00:00, 3374.14it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Binary Masking + Continent:\n", "F1: 92.08 Acc: 95.18\n", "F1 dif: 2.95 Acc dif: 0.03\n" ] } ], "source": [ "test_metadata['continent_binary'] = 0\n", "\n", "\n", "for index, row in tqdm.tqdm(test_metadata.iterrows(), total=len(test_metadata)):\n", " country = row.country\n", " continent = row.continent\n", " preds = sum((softmax(row.logits_t1), softmax(row.logits_t2), softmax(row.logits_t3))) / 3\n", " \n", " \n", " if country == 'Republic of Congo':\n", " country = 'republic of the congo'\n", "\n", " if country in ['unknown', 'West Bank', 'Macau S.A.R', 'US Naval Base Guantanamo Bay', 'United States Virgin Islands', 'Guam', 'Turks and Caicos Islands', 'Cyprus No Mans Area']:\n", " preds = preds\n", " else:\n", " preds = preds * np.array(country_relevance[country.lower()], dtype=np.int32)\n", "\n", "\n", " continent_dist = continent_distributions[continent]\n", " \n", " p_continent = (preds * continent_dist) / sum(preds * continent_dist)\n", " prior_ratio = p_continent / class_priors\n", " max_index = np.argmax(prior_ratio * preds) \n", " \n", " test_metadata.at[index, 'continent_binary'] = max_index\n", " \n", "f1 = f1_score(test_metadata['class_id'], test_metadata['continent_binary'], average='macro')\n", "accuracy = accuracy_score(test_metadata['class_id'], test_metadata['continent_binary'])\n", "\n", "print('Binary Masking + Continent:')\n", "print('F1:', np.round(f1 * 100, 2), 'Acc:', np.round(accuracy * 100, 2))\n", "print('F1 dif:', np.round((f1-vanilla_f1) * 100, 2), 'Acc dif:', np.round((accuracy-vanilla_accuracy) * 100, 2)) " ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "output = pd.DataFrame(zip(test_metadata.UUID, test_metadata.continent_binary), columns =['UUID', 'prediction'])\n", "output.to_csv('continent_binary.csv', index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.5" } }, "nbformat": 4, "nbformat_minor": 4 }