{ "cells": [ { "cell_type": "markdown", "id": "af313b4f", "metadata": {}, "source": [ "# Complexity Model Inference Test\n", "\n", "This notebook loads the saved production complexity model and its local artifacts from this folder, rebuilds the BiLSTM architecture, and runs inference on a few example sentences. It performs inference only." ] }, { "cell_type": "markdown", "id": "9976ef3c", "metadata": {}, "source": [ "## 1. Load the local artifacts\n", "\n", "The code below loads the checkpoint, model config, vocabulary, and label classes directly from the current folder." ] }, { "cell_type": "code", "execution_count": 1, "id": "635fe6e7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Model directory: /Users/georgijkutivadze/projects/mago-text-scoring/app/app-models/prod-model-complexity\n", "Device: cpu\n", "Artifacts: ['best.pt', 'label_classes.json', 'model-config.yaml', 'test_inference.ipynb', 'vocab.pkl']\n", "Checkpoint keys: ['epoch', 'model_state', 'optimizer_state', 'scheduler_state', 'best_f1', 'history', 'model_name', 'task']\n", "Embedding matrix shape: (50000, 300)\n", "Vocabulary size: 50000\n", "Labels: ['A1-A2', 'B1-B2', 'C1-C2']\n" ] } ], "source": [ "from pathlib import Path\n", "import json\n", "import pickle\n", "import re\n", "import unicodedata\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import torch\n", "import torch.nn as nn\n", "import yaml\n", "\n", "MODEL_DIR = Path(\".\").resolve()\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "\n", "with (MODEL_DIR / \"model-config.yaml\").open(\"r\", encoding=\"utf-8\") as handle:\n", " model_config = yaml.safe_load(handle)\n", "\n", "with (MODEL_DIR / \"label_classes.json\").open(\"r\", encoding=\"utf-8\") as handle:\n", " label_classes = json.load(handle)\n", "\n", "with (MODEL_DIR / \"vocab.pkl\").open(\"rb\") as handle:\n", " vocab = pickle.load(handle)\n", "\n", "checkpoint = torch.load(MODEL_DIR / \"best.pt\", map_location=device)\n", "state_dict = checkpoint[\"model_state\"] if isinstance(checkpoint, dict) and \"model_state\" in checkpoint else checkpoint\n", "embedding_matrix = state_dict[\"embedding.weight\"].detach().cpu().numpy()\n", "artifact_files = sorted(path.name for path in MODEL_DIR.iterdir() if path.is_file())\n", "\n", "print(f\"Model directory: {MODEL_DIR}\")\n", "print(f\"Device: {device}\")\n", "print(f\"Artifacts: {artifact_files}\")\n", "print(f\"Checkpoint keys: {list(checkpoint.keys()) if isinstance(checkpoint, dict) else 'state_dict'}\")\n", "print(f\"Embedding matrix shape: {embedding_matrix.shape}\")\n", "print(f\"Vocabulary size: {len(vocab)}\")\n", "print(f\"Labels: {label_classes}\")\n" ] }, { "cell_type": "markdown", "id": "abe5a2e3", "metadata": {}, "source": [ "## 2. Recreate the model and preprocessing pipeline\n", "\n", "The notebook rebuilds the saved BiLSTM + additive-attention classifier and applies the lightweight text normalization used for inference." ] }, { "cell_type": "code", "execution_count": 2, "id": "f86fa8a2", "metadata": {}, "outputs": [], "source": [ "class AdditiveAttention(nn.Module):\n", " def __init__(self, hidden_dim: int):\n", " super().__init__()\n", " self.attn = nn.Linear(hidden_dim, 1, bias=False)\n", "\n", " def forward(self, hidden_states, mask=None):\n", " scores = self.attn(hidden_states).squeeze(-1)\n", " if mask is not None:\n", " scores = scores.masked_fill(~mask, -1e9)\n", " weights = torch.softmax(scores, dim=1)\n", " context = (weights.unsqueeze(-1) * hidden_states).sum(dim=1)\n", " return context, weights\n", "\n", "\n", "class BiLSTMAttention(nn.Module):\n", " def __init__(\n", " self,\n", " embedding_matrix: np.ndarray,\n", " num_classes: int,\n", " hidden_dim: int = 128,\n", " num_layers: int = 2,\n", " dropout: float = 0.3,\n", " ):\n", " super().__init__()\n", " _, embedding_dim = embedding_matrix.shape\n", " self.embedding = nn.Embedding.from_pretrained(\n", " torch.tensor(embedding_matrix, dtype=torch.float32),\n", " freeze=False,\n", " padding_idx=0,\n", " )\n", " self.lstm = nn.LSTM(\n", " input_size=embedding_dim,\n", " hidden_size=hidden_dim,\n", " num_layers=num_layers,\n", " batch_first=True,\n", " bidirectional=True,\n", " dropout=dropout if num_layers > 1 else 0.0,\n", " )\n", " self.attention = AdditiveAttention(hidden_dim * 2)\n", " self.dropout = nn.Dropout(dropout)\n", " self.fc = nn.Linear(hidden_dim * 2, num_classes)\n", "\n", " def forward(self, token_ids):\n", " mask = token_ids != 0\n", " embeddings = self.dropout(self.embedding(token_ids))\n", " hidden_states, _ = self.lstm(embeddings)\n", " context, _ = self.attention(hidden_states, mask)\n", " return self.fc(self.dropout(context))\n", "\n", "\n", "DEFAULT_STOPWORDS = {\n", " \"a\", \"an\", \"and\", \"are\", \"as\", \"at\", \"be\", \"by\", \"for\", \"from\",\n", " \"has\", \"he\", \"in\", \"is\", \"it\", \"its\", \"of\", \"on\", \"that\", \"the\",\n", " \"to\", \"was\", \"were\", \"will\", \"with\"\n", "}\n", "\n", "try:\n", " from nltk.corpus import stopwords\n", " STOPWORDS = set(stopwords.words(\"english\"))\n", "except Exception:\n", " STOPWORDS = DEFAULT_STOPWORDS\n", "\n", "try:\n", " from nltk.stem import WordNetLemmatizer\n", " LEMMATIZER = WordNetLemmatizer()\n", "except Exception:\n", " LEMMATIZER = None\n", "\n", "\n", "def maybe_lemmatize(token: str) -> str:\n", " if LEMMATIZER is None:\n", " return token\n", " try:\n", " return LEMMATIZER.lemmatize(token)\n", " except Exception:\n", " return token\n", "\n", "\n", "def normalize_text(text: str) -> str:\n", " text = str(text).lower()\n", " text = unicodedata.normalize(\"NFKD\", text)\n", " text = re.sub(r\"http\\S+|www\\S+\", \" \", text)\n", " text = re.sub(r\"@\\w+|#\\w+\", \" \", text)\n", " text = re.sub(r\"[^a-zA-Z0-9\\s!?.,'\\\"]\", \" \", text)\n", " text = re.sub(r\"\\s+\", \" \", text).strip()\n", " return text\n", "\n", "\n", "def preprocess_from_normalized(text: str) -> str:\n", " tokens = [token for token in text.split() if token not in STOPWORDS]\n", " return \" \".join(maybe_lemmatize(token) for token in tokens)\n", "\n", "\n", "def texts_to_sequences(texts, vocab_map, max_len, unk_token=\"\"):\n", " unk_idx = vocab_map.get(unk_token, 1)\n", " sequences = np.zeros((len(texts), max_len), dtype=np.int64)\n", " for row_idx, text in enumerate(texts):\n", " for col_idx, token in enumerate(str(text).split()[:max_len]):\n", " sequences[row_idx, col_idx] = vocab_map.get(token, unk_idx)\n", " return sequences\n" ] }, { "cell_type": "markdown", "id": "0c36b0e4", "metadata": {}, "source": [ "## 3. Restore the checkpoint and run example inference\n", "\n", "This cell rebuilds the model, restores the saved weights, preprocesses example sentences, and displays the predicted complexity label with probabilities." ] }, { "cell_type": "code", "execution_count": 5, "id": "229ce4ed", "metadata": {}, "outputs": [], "source": [ "runtime_config = model_config.get(\"runtime\", {})\n", "network_config = model_config.get(\"model\", {})\n", "\n", "model = BiLSTMAttention(\n", " embedding_matrix=embedding_matrix,\n", " num_classes=len(label_classes),\n", " hidden_dim=network_config.get(\"hidden_dim\", 128),\n", " num_layers=network_config.get(\"num_layers\", 2),\n", " dropout=network_config.get(\"dropout\", 0.3),\n", ")\n", "model.load_state_dict(state_dict)\n", "model.to(device)\n", "model.eval()\n", "\n", "MAX_LEN = runtime_config.get(\"max_sequence_length\", 256)\n", "\n", "\n", "@torch.inference_mode()\n", "def predict_texts(texts):\n", " normalized = [normalize_text(text) for text in texts]\n", " preprocessed = [preprocess_from_normalized(text) for text in normalized]\n", " sequences = texts_to_sequences(preprocessed, vocab, MAX_LEN)\n", " inputs = torch.tensor(sequences, dtype=torch.long, device=device)\n", " logits = model(inputs)\n", " probabilities = torch.softmax(logits, dim=-1).cpu().numpy()\n", "\n", " rows = []\n", " for raw_text, normalized_text, preprocessed_text, probs in zip(texts, normalized, preprocessed, probabilities):\n", " pred_idx = int(probs.argmax())\n", " rows.append(\n", " {\n", " \"text\": raw_text,\n", " \"normalized_text\": normalized_text,\n", " \"preprocessed_text\": preprocessed_text,\n", " \"predicted_label\": label_classes[pred_idx],\n", " \"confidence\": round(float(probs[pred_idx]), 4),\n", " \"probabilities\": {\n", " label: round(float(score), 4)\n", " for label, score in zip(label_classes, probs)\n", " },\n", " }\n", " )\n", "\n", " return pd.DataFrame(rows)" ] }, { "cell_type": "code", "execution_count": 7, "id": "62c8eeb8-3663-40f7-9b75-20ba0e6017db", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
textnormalized_textpreprocessed_textpredicted_labelconfidenceprobabilities
0I have a dog. It is small and very cute. We play in the park every day.i have a dog. it is small and very cute. we play in the park every day.dog. small cute. play park every day.A1-A20.6027{'A1-A2': 0.6027, 'B1-B2': 0.3619, 'C1-C2': 0.0354}
1My name is Anna. I am a student. I live with my family in a big city.my name is anna. i am a student. i live with my family in a big city.name anna. student. live family big city.C1-C20.7338{'A1-A2': 0.1452, 'B1-B2': 0.121, 'C1-C2': 0.7338}
2Many people enjoy traveling by train because it is comfortable, affordable, and less stressful than flying.many people enjoy traveling by train because it is comfortable, affordable, and less stressful than flying.many people enjoy traveling train comfortable, affordable, less stressful flying.C1-C20.8663{'A1-A2': 0.0018, 'B1-B2': 0.1319, 'C1-C2': 0.8663}
3If students manage their time well, they can improve their grades and still have enough time for rest and hobbies.if students manage their time well, they can improve their grades and still have enough time for rest and hobbies.student manage time well, improve grade still enough time rest hobbies.B1-B20.9152{'A1-A2': 0.0168, 'B1-B2': 0.9152, 'C1-C2': 0.068}
4The committee’s recommendation, while ostensibly pragmatic, ultimately fails to address the structural asymmetries t...the committee s recommendation, while ostensibly pragmatic, ultimately fails to address the structural asymmetries t...committee recommendation, ostensibly pragmatic, ultimately fails address structural asymmetry perpetuate institution...C1-C20.9565{'A1-A2': 0.0023, 'B1-B2': 0.0412, 'C1-C2': 0.9565}
5Her argument is particularly compelling because it synthesizes historical evidence, theoretical precision, and a sop...her argument is particularly compelling because it synthesizes historical evidence, theoretical precision, and a sop...argument particularly compelling synthesizes historical evidence, theoretical precision, sophisticated awareness cul...B1-B20.6361{'A1-A2': 0.2443, 'B1-B2': 0.6361, 'C1-C2': 0.1197}
\n", "
" ], "text/plain": [ " text \\\n", "0 I have a dog. It is small and very cute. We play in the park every day. \n", "1 My name is Anna. I am a student. I live with my family in a big city. \n", "2 Many people enjoy traveling by train because it is comfortable, affordable, and less stressful than flying. \n", "3 If students manage their time well, they can improve their grades and still have enough time for rest and hobbies. \n", "4 The committee’s recommendation, while ostensibly pragmatic, ultimately fails to address the structural asymmetries t... \n", "5 Her argument is particularly compelling because it synthesizes historical evidence, theoretical precision, and a sop... \n", "\n", " normalized_text \\\n", "0 i have a dog. it is small and very cute. we play in the park every day. \n", "1 my name is anna. i am a student. i live with my family in a big city. \n", "2 many people enjoy traveling by train because it is comfortable, affordable, and less stressful than flying. \n", "3 if students manage their time well, they can improve their grades and still have enough time for rest and hobbies. \n", "4 the committee s recommendation, while ostensibly pragmatic, ultimately fails to address the structural asymmetries t... \n", "5 her argument is particularly compelling because it synthesizes historical evidence, theoretical precision, and a sop... \n", "\n", " preprocessed_text \\\n", "0 dog. small cute. play park every day. \n", "1 name anna. student. live family big city. \n", "2 many people enjoy traveling train comfortable, affordable, less stressful flying. \n", "3 student manage time well, improve grade still enough time rest hobbies. \n", "4 committee recommendation, ostensibly pragmatic, ultimately fails address structural asymmetry perpetuate institution... \n", "5 argument particularly compelling synthesizes historical evidence, theoretical precision, sophisticated awareness cul... \n", "\n", " predicted_label confidence \\\n", "0 A1-A2 0.6027 \n", "1 C1-C2 0.7338 \n", "2 C1-C2 0.8663 \n", "3 B1-B2 0.9152 \n", "4 C1-C2 0.9565 \n", "5 B1-B2 0.6361 \n", "\n", " probabilities \n", "0 {'A1-A2': 0.6027, 'B1-B2': 0.3619, 'C1-C2': 0.0354} \n", "1 {'A1-A2': 0.1452, 'B1-B2': 0.121, 'C1-C2': 0.7338} \n", "2 {'A1-A2': 0.0018, 'B1-B2': 0.1319, 'C1-C2': 0.8663} \n", "3 {'A1-A2': 0.0168, 'B1-B2': 0.9152, 'C1-C2': 0.068} \n", "4 {'A1-A2': 0.0023, 'B1-B2': 0.0412, 'C1-C2': 0.9565} \n", "5 {'A1-A2': 0.2443, 'B1-B2': 0.6361, 'C1-C2': 0.1197} " ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd.set_option(\"display.max_colwidth\", 120)\n", "\n", "example_sentences = [\n", " # A1-A2\n", " \"I have a dog. It is small and very cute. We play in the park every day.\",\n", " \"My name is Anna. I am a student. I live with my family in a big city.\",\n", "\n", " # B1-B2\n", " \"Many people enjoy traveling by train because it is comfortable, affordable, and less stressful than flying.\",\n", " \"If students manage their time well, they can improve their grades and still have enough time for rest and hobbies.\",\n", "\n", " # C1-C2\n", " \"The committee’s recommendation, while ostensibly pragmatic, ultimately fails to address the structural asymmetries that perpetuate institutional inefficiency.\",\n", " \"Her argument is particularly compelling because it synthesizes historical evidence, theoretical precision, and a sophisticated awareness of cultural nuance.\"\n", "]\n", "\n", "results = predict_texts(example_sentences)\n", "results" ] }, { "cell_type": "code", "execution_count": null, "id": "31f6fc98-7438-4e16-a8aa-87e63c61ee6e", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.11" } }, "nbformat": 4, "nbformat_minor": 5 }