Upload folder using huggingface_hub

.ipynb_checkpoints/test_inference-checkpoint.ipynb

@@ -0,0 +1,277 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Complexity Model Inference Test\n",
+    "\n",
+    "This notebook loads the saved production complexity model and its local artifacts from this folder, rebuilds the BiLSTM architecture, and runs inference on a few example sentences. It performs inference only."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 1. Load the local artifacts\n",
+    "\n",
+    "The code below loads the checkpoint, model config, vocabulary, and label classes directly from the current folder."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "import json\n",
+    "import pickle\n",
+    "import re\n",
+    "import unicodedata\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import yaml\n",
+    "\n",
+    "MODEL_DIR = Path(\".\").resolve()\n",
+    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "\n",
+    "with (MODEL_DIR / \"model-config.yaml\").open(\"r\", encoding=\"utf-8\") as handle:\n",
+    "    model_config = yaml.safe_load(handle)\n",
+    "\n",
+    "with (MODEL_DIR / \"label_classes.json\").open(\"r\", encoding=\"utf-8\") as handle:\n",
+    "    label_classes = json.load(handle)\n",
+    "\n",
+    "with (MODEL_DIR / \"vocab.pkl\").open(\"rb\") as handle:\n",
+    "    vocab = pickle.load(handle)\n",
+    "\n",
+    "checkpoint = torch.load(MODEL_DIR / \"best.pt\", map_location=device)\n",
+    "state_dict = checkpoint[\"model_state\"] if isinstance(checkpoint, dict) and \"model_state\" in checkpoint else checkpoint\n",
+    "embedding_matrix = state_dict[\"embedding.weight\"].detach().cpu().numpy()\n",
+    "artifact_files = sorted(path.name for path in MODEL_DIR.iterdir() if path.is_file())\n",
+    "\n",
+    "print(f\"Model directory: {MODEL_DIR}\")\n",
+    "print(f\"Device: {device}\")\n",
+    "print(f\"Artifacts: {artifact_files}\")\n",
+    "print(f\"Checkpoint keys: {list(checkpoint.keys()) if isinstance(checkpoint, dict) else 'state_dict'}\")\n",
+    "print(f\"Embedding matrix shape: {embedding_matrix.shape}\")\n",
+    "print(f\"Vocabulary size: {len(vocab)}\")\n",
+    "print(f\"Labels: {label_classes}\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. Recreate the model and preprocessing pipeline\n",
+    "\n",
+    "The notebook rebuilds the saved BiLSTM + additive-attention classifier and applies the lightweight text normalization used for inference."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class AdditiveAttention(nn.Module):\n",
+    "    def __init__(self, hidden_dim: int):\n",
+    "        super().__init__()\n",
+    "        self.attn = nn.Linear(hidden_dim, 1, bias=False)\n",
+    "\n",
+    "    def forward(self, hidden_states, mask=None):\n",
+    "        scores = self.attn(hidden_states).squeeze(-1)\n",
+    "        if mask is not None:\n",
+    "            scores = scores.masked_fill(~mask, -1e9)\n",
+    "        weights = torch.softmax(scores, dim=1)\n",
+    "        context = (weights.unsqueeze(-1) * hidden_states).sum(dim=1)\n",
+    "        return context, weights\n",
+    "\n",
+    "\n",
+    "class BiLSTMAttention(nn.Module):\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        embedding_matrix: np.ndarray,\n",
+    "        num_classes: int,\n",
+    "        hidden_dim: int = 128,\n",
+    "        num_layers: int = 2,\n",
+    "        dropout: float = 0.3,\n",
+    "    ):\n",
+    "        super().__init__()\n",
+    "        _, embedding_dim = embedding_matrix.shape\n",
+    "        self.embedding = nn.Embedding.from_pretrained(\n",
+    "            torch.tensor(embedding_matrix, dtype=torch.float32),\n",
+    "            freeze=False,\n",
+    "            padding_idx=0,\n",
+    "        )\n",
+    "        self.lstm = nn.LSTM(\n",
+    "            input_size=embedding_dim,\n",
+    "            hidden_size=hidden_dim,\n",
+    "            num_layers=num_layers,\n",
+    "            batch_first=True,\n",
+    "            bidirectional=True,\n",
+    "            dropout=dropout if num_layers > 1 else 0.0,\n",
+    "        )\n",
+    "        self.attention = AdditiveAttention(hidden_dim * 2)\n",
+    "        self.dropout = nn.Dropout(dropout)\n",
+    "        self.fc = nn.Linear(hidden_dim * 2, num_classes)\n",
+    "\n",
+    "    def forward(self, token_ids):\n",
+    "        mask = token_ids != 0\n",
+    "        embeddings = self.dropout(self.embedding(token_ids))\n",
+    "        hidden_states, _ = self.lstm(embeddings)\n",
+    "        context, _ = self.attention(hidden_states, mask)\n",
+    "        return self.fc(self.dropout(context))\n",
+    "\n",
+    "\n",
+    "DEFAULT_STOPWORDS = {\n",
+    "    \"a\", \"an\", \"and\", \"are\", \"as\", \"at\", \"be\", \"by\", \"for\", \"from\",\n",
+    "    \"has\", \"he\", \"in\", \"is\", \"it\", \"its\", \"of\", \"on\", \"that\", \"the\",\n",
+    "    \"to\", \"was\", \"were\", \"will\", \"with\"\n",
+    "}\n",
+    "\n",
+    "try:\n",
+    "    from nltk.corpus import stopwords\n",
+    "    STOPWORDS = set(stopwords.words(\"english\"))\n",
+    "except Exception:\n",
+    "    STOPWORDS = DEFAULT_STOPWORDS\n",
+    "\n",
+    "try:\n",
+    "    from nltk.stem import WordNetLemmatizer\n",
+    "    LEMMATIZER = WordNetLemmatizer()\n",
+    "except Exception:\n",
+    "    LEMMATIZER = None\n",
+    "\n",
+    "\n",
+    "def maybe_lemmatize(token: str) -> str:\n",
+    "    if LEMMATIZER is None:\n",
+    "        return token\n",
+    "    try:\n",
+    "        return LEMMATIZER.lemmatize(token)\n",
+    "    except Exception:\n",
+    "        return token\n",
+    "\n",
+    "\n",
+    "def normalize_text(text: str) -> str:\n",
+    "    text = str(text).lower()\n",
+    "    text = unicodedata.normalize(\"NFKD\", text)\n",
+    "    text = re.sub(r\"http\\S+|www\\S+\", \" \", text)\n",
+    "    text = re.sub(r\"@\\w+|#\\w+\", \" \", text)\n",
+    "    text = re.sub(r\"[^a-zA-Z0-9\\s!?.,'\\\"]\", \" \", text)\n",
+    "    text = re.sub(r\"\\s+\", \" \", text).strip()\n",
+    "    return text\n",
+    "\n",
+    "\n",
+    "def preprocess_from_normalized(text: str) -> str:\n",
+    "    tokens = [token for token in text.split() if token not in STOPWORDS]\n",
+    "    return \" \".join(maybe_lemmatize(token) for token in tokens)\n",
+    "\n",
+    "\n",
+    "def texts_to_sequences(texts, vocab_map, max_len, unk_token=\"<UNK>\"):\n",
+    "    unk_idx = vocab_map.get(unk_token, 1)\n",
+    "    sequences = np.zeros((len(texts), max_len), dtype=np.int64)\n",
+    "    for row_idx, text in enumerate(texts):\n",
+    "        for col_idx, token in enumerate(str(text).split()[:max_len]):\n",
+    "            sequences[row_idx, col_idx] = vocab_map.get(token, unk_idx)\n",
+    "    return sequences\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 3. Restore the checkpoint and run example inference\n",
+    "\n",
+    "This cell rebuilds the model, restores the saved weights, preprocesses example sentences, and displays the predicted complexity label with probabilities."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "runtime_config = model_config.get(\"runtime\", {})\n",
+    "network_config = model_config.get(\"model\", {})\n",
+    "\n",
+    "model = BiLSTMAttention(\n",
+    "    embedding_matrix=embedding_matrix,\n",
+    "    num_classes=len(label_classes),\n",
+    "    hidden_dim=network_config.get(\"hidden_dim\", 128),\n",
+    "    num_layers=network_config.get(\"num_layers\", 2),\n",
+    "    dropout=network_config.get(\"dropout\", 0.3),\n",
+    ")\n",
+    "model.load_state_dict(state_dict)\n",
+    "model.to(device)\n",
+    "model.eval()\n",
+    "\n",
+    "MAX_LEN = runtime_config.get(\"max_sequence_length\", 256)\n",
+    "\n",
+    "\n",
+    "@torch.inference_mode()\n",
+    "def predict_texts(texts):\n",
+    "    normalized = [normalize_text(text) for text in texts]\n",
+    "    preprocessed = [preprocess_from_normalized(text) for text in normalized]\n",
+    "    sequences = texts_to_sequences(preprocessed, vocab, MAX_LEN)\n",
+    "    inputs = torch.tensor(sequences, dtype=torch.long, device=device)\n",
+    "    logits = model(inputs)\n",
+    "    probabilities = torch.softmax(logits, dim=-1).cpu().numpy()\n",
+    "\n",
+    "    rows = []\n",
+    "    for raw_text, normalized_text, preprocessed_text, probs in zip(texts, normalized, preprocessed, probabilities):\n",
+    "        pred_idx = int(probs.argmax())\n",
+    "        rows.append(\n",
+    "            {\n",
+    "                \"text\": raw_text,\n",
+    "                \"normalized_text\": normalized_text,\n",
+    "                \"preprocessed_text\": preprocessed_text,\n",
+    "                \"predicted_label\": label_classes[pred_idx],\n",
+    "                \"confidence\": round(float(probs[pred_idx]), 4),\n",
+    "                \"probabilities\": {\n",
+    "                    label: round(float(score), 4)\n",
+    "                    for label, score in zip(label_classes, probs)\n",
+    "                },\n",
+    "            }\n",
+    "        )\n",
+    "\n",
+    "    return pd.DataFrame(rows)\n",
+    "\n",
+    "\n",
+    "example_sentences = [\n",
+    "    \"The cat sat on the mat.\",\n",
+    "    \"Students should explain the main idea in clear sentences.\",\n",
+    "    \"Although the study is limited, it still offers a useful interpretation of the data.\",\n",
+    "    \"Notwithstanding several methodological caveats, the argument articulates a nuanced account of institutional change.\",\n",
+    "]\n",
+    "\n",
+    "pd.set_option(\"display.max_colwidth\", 120)\n",
+    "results = predict_texts(example_sentences)\n",
+    "results\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "nlp-project",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf01e442856c2587c087c4a9cca1ccb9549bd757a4755a72e2d75038aa998d8a
+size 190059249

dashboard/curves/training-history.json

@@ -0,0 +1,66 @@
+{
+  "source_file": "complexity/logs/complexity-classification_GloVe_BiLSTM_[train_pool].log",
+  "x_axis": "epoch",
+  "points": [
+    {
+      "epoch": 1,
+      "total_epochs": 20,
+      "train_loss": 0.8229,
+      "val_loss": 0.7119,
+      "val_f1_macro": 0.6324,
+      "epoch_seconds": 79.1
+    },
+    {
+      "epoch": 2,
+      "total_epochs": 20,
+      "train_loss": 0.488,
+      "val_loss": 0.6723,
+      "val_f1_macro": 0.655,
+      "epoch_seconds": 83.1
+    },
+    {
+      "epoch": 3,
+      "total_epochs": 20,
+      "train_loss": 0.2493,
+      "val_loss": 0.8044,
+      "val_f1_macro": 0.6605,
+      "epoch_seconds": 83.2
+    },
+    {
+      "epoch": 4,
+      "total_epochs": 20,
+      "train_loss": 0.116,
+      "val_loss": 1.2446,
+      "val_f1_macro": 0.6473,
+      "epoch_seconds": 82.7
+    },
+    {
+      "epoch": 5,
+      "total_epochs": 20,
+      "train_loss": 0.0496,
+      "val_loss": 1.5788,
+      "val_f1_macro": 0.6302,
+      "epoch_seconds": 80.7
+    },
+    {
+      "epoch": 6,
+      "total_epochs": 20,
+      "train_loss": 0.0151,
+      "val_loss": 1.9807,
+      "val_f1_macro": 0.6283,
+      "epoch_seconds": 79.6
+    }
+  ],
+  "best_eval_event": {
+    "epoch": 3,
+    "total_epochs": 20,
+    "train_loss": 0.2493,
+    "val_loss": 0.8044,
+    "val_f1_macro": 0.6605,
+    "epoch_seconds": 83.2
+  },
+  "notes": [
+    "Early stopping triggered at epoch 6.",
+    "Restored best checkpoint | val_f1=0.6605 | path=models/glove_bilstm_complexity_train_pool_20260307_163358/best_model/best.pt"
+  ]
+}

dashboard/dashboard-manifest.json

@@ -0,0 +1,187 @@
+{
+  "schema_version": "1.0.0",
+  "generated_at": "2026-03-12T17:02:22+00:00",
+  "bundle_builder": "generic-v1",
+  "dashboard_root": "app/app-models/prod-model-complexity/dashboard",
+  "model": {
+    "model_id": "complexity-bilstm-final-v1",
+    "display_name": "Complexity BiLSTM Final v1",
+    "domain": "complexity",
+    "description": "Final BiLSTM with additive attention for CEFR-based text complexity classification."
+  },
+  "entrypoints": {
+    "overview": "app/app-models/prod-model-complexity/dashboard/summary/overview.json",
+    "source_audit": "app/app-models/prod-model-complexity/dashboard/summary/source-audit.json"
+  },
+  "sections": [
+    {
+      "id": "metadata",
+      "title": "Model Metadata",
+      "status": "available",
+      "description": "Model identity, runtime settings, and optional experiment configuration.",
+      "files": [
+        "app/app-models/prod-model-complexity/dashboard/metadata/model.json",
+        "app/app-models/prod-model-complexity/dashboard/metadata/experiment-config.json"
+      ]
+    },
+    {
+      "id": "summary",
+      "title": "Summary",
+      "status": "available",
+      "description": "Dashboard status overview and source audit for the generated bundle.",
+      "files": [
+        "app/app-models/prod-model-complexity/dashboard/summary/overview.json",
+        "app/app-models/prod-model-complexity/dashboard/summary/source-audit.json"
+      ]
+    },
+    {
+      "id": "evaluation",
+      "title": "Primary Evaluation",
+      "status": "available",
+      "description": "Primary validation and test metrics for the production model.",
+      "files": [
+        "app/app-models/prod-model-complexity/dashboard/metrics/primary-evaluation.json"
+      ]
+    },
+    {
+      "id": "benchmark",
+      "title": "Benchmark",
+      "status": "available",
+      "description": "Comparison rows that place the production model in the wider benchmark context.",
+      "files": [
+        "app/app-models/prod-model-complexity/dashboard/metrics/benchmark-test.json",
+        "app/app-models/prod-model-complexity/dashboard/figures/benchmark-test-f1.plotly.json"
+      ],
+      "charts": [
+        "benchmark-test-f1"
+      ]
+    },
+    {
+      "id": "training_curves",
+      "title": "Training Curves",
+      "status": "available",
+      "description": "Training history exported from the model training run.",
+      "files": [
+        "app/app-models/prod-model-complexity/dashboard/curves/training-history.json",
+        "app/app-models/prod-model-complexity/dashboard/figures/training-loss.plotly.json",
+        "app/app-models/prod-model-complexity/dashboard/figures/eval-metrics.plotly.json"
+      ],
+      "charts": [
+        "training-loss",
+        "eval-metrics"
+      ]
+    },
+    {
+      "id": "learning_curves",
+      "title": "Learning Curves",
+      "status": "not_applicable",
+      "description": "Sample-size or checkpoint learning-curve artifacts for the domain.",
+      "files": [],
+      "reason": "The complexity benchmark is built around a fixed train_pool dataset rather than multiple prod-relevant train sizes."
+    },
+    {
+      "id": "cross_dataset",
+      "title": "Cross Dataset Evaluation",
+      "status": "available",
+      "description": "Evaluation rows produced on datasets outside the primary split.",
+      "files": [
+        "app/app-models/prod-model-complexity/dashboard/metrics/cross-dataset.json",
+        "app/app-models/prod-model-complexity/dashboard/figures/cross-dataset-f1.plotly.json"
+      ],
+      "charts": [
+        "cross-dataset-f1"
+      ]
+    },
+    {
+      "id": "confusion_matrix",
+      "title": "Confusion Matrix",
+      "status": "image_only",
+      "description": "Static confusion-matrix image assets copied into the dashboard bundle.",
+      "files": [
+        "app/app-models/prod-model-complexity/dashboard/confusion/cm-glove-bilstm-train-pool.png"
+      ]
+    },
+    {
+      "id": "class_distribution",
+      "title": "Class Distribution",
+      "status": "missing",
+      "description": "Class and optional source-dataset distributions derived from explicit split inputs.",
+      "files": [],
+      "reason": "No class_distribution source was configured."
+    },
+    {
+      "id": "samples",
+      "title": "Prediction Samples",
+      "status": "available",
+      "description": "Prediction examples for qualitative inspection.",
+      "files": [
+        "app/app-models/prod-model-complexity/dashboard/samples/prediction-samples.json"
+      ]
+    }
+  ],
+  "selected_sources": [
+    {
+      "category": "metadata",
+      "path": "app/app-models/prod-model-complexity/model-config.yaml",
+      "reason": "Production model identity, runtime settings, and label mapping."
+    },
+    {
+      "category": "experiment_config",
+      "path": "complexity/outputs/metrics/cfg_20260310_2125.json",
+      "reason": "Experiment or evaluation config attached for traceability."
+    },
+    {
+      "category": "primary_evaluation",
+      "path": "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_latest.json",
+      "reason": "Primary validation/test metrics for the production model."
+    },
+    {
+      "category": "benchmark",
+      "path": "complexity/outputs/metrics/results_complexity-classification.csv",
+      "reason": "Benchmark rows used to compare the production model against peers."
+    },
+    {
+      "category": "training_history",
+      "path": "complexity/logs/complexity-classification_GloVe_BiLSTM_[train_pool].log",
+      "reason": "Training-history artifact used to build learning and loss curves."
+    },
+    {
+      "category": "cross_dataset",
+      "path": "complexity/outputs/cross_dataset_eval/summary_latest.csv",
+      "reason": "Cross-dataset evaluation summary rows."
+    },
+    {
+      "category": "class_distribution",
+      "path": "complexity/data/train_pool_train.parquet",
+      "reason": "Explicit split artifacts used to derive class distributions."
+    },
+    {
+      "category": "class_distribution",
+      "path": "complexity/data/train_pool_val.parquet",
+      "reason": "Explicit split artifacts used to derive class distributions."
+    },
+    {
+      "category": "class_distribution",
+      "path": "complexity/data/train_pool_test.parquet",
+      "reason": "Explicit split artifacts used to derive class distributions."
+    },
+    {
+      "category": "prediction_samples",
+      "path": "complexity/outputs/metrics/manual_example_predictions.csv",
+      "reason": "Prediction samples used for qualitative dashboard review."
+    },
+    {
+      "category": "confusion_matrix",
+      "path": "complexity/outputs/plots/confusion_matrices/cm_glove_bilstm_train_pool.png",
+      "reason": "Confusion-matrix image copied into the dashboard bundle."
+    }
+  ],
+  "notes": [
+    "Primary evaluation uses the production GloVe BiLSTM metrics_latest.json referenced by prod-model-complexity/model-config.yaml.",
+    "Complexity is the richest dashboard domain in the repo: internal benchmark data, external evaluation summaries, split parquets, manual sample predictions, and prod-model training logs are all present.",
+    "Confusion matrices exist only as saved images for the production model; no numeric matrix artifact was found.",
+    "class_distribution parsing failed for complexity/data/train_pool_train.parquet: Parquet dashboard sources require the optional pandas dependency.",
+    "class_distribution parsing failed for complexity/data/train_pool_val.parquet: Parquet dashboard sources require the optional pandas dependency.",
+    "class_distribution parsing failed for complexity/data/train_pool_test.parquet: Parquet dashboard sources require the optional pandas dependency."
+  ]
+}

dashboard/distributions/class-distribution.json

@@ -0,0 +1,78 @@
+{
+  "overall": [
+    {
+      "split": "overall",
+      "label": "A1-A2",
+      "count": 13598,
+      "share": 0.304049
+    },
+    {
+      "split": "overall",
+      "label": "B1-B2",
+      "count": 24366,
+      "share": 0.54482
+    },
+    {
+      "split": "overall",
+      "label": "C1-C2",
+      "count": 6759,
+      "share": 0.15113
+    }
+  ],
+  "splits": [
+    {
+      "split": "train",
+      "label": "A1-A2",
+      "count": 9518,
+      "share": 0.304041
+    },
+    {
+      "split": "train",
+      "label": "B1-B2",
+      "count": 17056,
+      "share": 0.544833
+    },
+    {
+      "split": "train",
+      "label": "C1-C2",
+      "count": 4731,
+      "share": 0.151126
+    },
+    {
+      "split": "val",
+      "label": "A1-A2",
+      "count": 2040,
+      "share": 0.304069
+    },
+    {
+      "split": "val",
+      "label": "B1-B2",
+      "count": 3655,
+      "share": 0.544791
+    },
+    {
+      "split": "val",
+      "label": "C1-C2",
+      "count": 1014,
+      "share": 0.15114
+    },
+    {
+      "split": "test",
+      "label": "A1-A2",
+      "count": 2040,
+      "share": 0.304069
+    },
+    {
+      "split": "test",
+      "label": "B1-B2",
+      "count": 3655,
+      "share": 0.544791
+    },
+    {
+      "split": "test",
+      "label": "C1-C2",
+      "count": 1014,
+      "share": 0.15114
+    }
+  ]
+}

dashboard/distributions/source-dataset-distribution.json

@@ -0,0 +1,54 @@
+{
+  "overall": [
+    {
+      "split": "overall",
+      "source_dataset": "asap-2-0",
+      "count": 24728,
+      "share": 0.552915
+    },
+    {
+      "split": "overall",
+      "source_dataset": "uk-key-stage-readability-for-english-texts",
+      "count": 19995,
+      "share": 0.447085
+    }
+  ],
+  "splits": [
+    {
+      "split": "train",
+      "source_dataset": "asap-2-0",
+      "count": 17363,
+      "share": 0.55464
+    },
+    {
+      "split": "train",
+      "source_dataset": "uk-key-stage-readability-for-english-texts",
+      "count": 13942,
+      "share": 0.44536
+    },
+    {
+      "split": "val",
+      "source_dataset": "asap-2-0",
+      "count": 3684,
+      "share": 0.549113
+    },
+    {
+      "split": "val",
+      "source_dataset": "uk-key-stage-readability-for-english-texts",
+      "count": 3025,
+      "share": 0.450887
+    },
+    {
+      "split": "test",
+      "source_dataset": "asap-2-0",
+      "count": 3681,
+      "share": 0.548666
+    },
+    {
+      "split": "test",
+      "source_dataset": "uk-key-stage-readability-for-english-texts",
+      "count": 3028,
+      "share": 0.451334
+    }
+  ]
+}

dashboard/figures/benchmark-test-f1.plotly.json

@@ -0,0 +1,77 @@
+{
+  "data": [
+    {
+      "type": "bar",
+      "orientation": "v",
+      "x": [
+        "DeBERTaV3 train pool",
+        "DistilBERT train pool",
+        "RoBERTa train pool",
+        "TF-IDF + SVC [train pool]",
+        "TF-IDF + LogReg [train pool]",
+        "GloVe BiLSTM [train pool]",
+        "GloVe CNN [train pool]",
+        "GloVe MLP [train pool]",
+        "TF-IDF + XGBoost [train pool]",
+        "Majority Class [train pool]"
+      ],
+      "y": [
+        0.7510812492382728,
+        0.7430667400906253,
+        0.7297663021875974,
+        0.6686789738725382,
+        0.6607874127954448,
+        0.6551661294897166,
+        0.6388266463257223,
+        0.5964460940300637,
+        0.534220517963146,
+        0.2351087096359192
+      ],
+      "marker": {
+        "color": "#0f766e"
+      },
+      "text": [
+        "0.751",
+        "0.743",
+        "0.730",
+        "0.669",
+        "0.661",
+        "0.655",
+        "0.639",
+        "0.596",
+        "0.534",
+        "0.235"
+      ],
+      "textposition": "auto"
+    }
+  ],
+  "layout": {
+    "title": {
+      "text": "Benchmark Comparison"
+    },
+    "paper_bgcolor": "white",
+    "plot_bgcolor": "white",
+    "font": {
+      "family": "Arial, sans-serif",
+      "size": 12
+    },
+    "margin": {
+      "l": 60,
+      "r": 20,
+      "t": 60,
+      "b": 60
+    },
+    "xaxis": {
+      "title": {
+        "text": "Model"
+      },
+      "automargin": true
+    },
+    "yaxis": {
+      "title": {
+        "text": "F1 Macro"
+      },
+      "automargin": true
+    }
+  }
+}

dashboard/figures/class-distribution.plotly.json

@@ -0,0 +1,56 @@
+{
+  "data": [
+    {
+      "type": "bar",
+      "orientation": "v",
+      "x": [
+        "A1-A2",
+        "B1-B2",
+        "C1-C2"
+      ],
+      "y": [
+        13598,
+        24366,
+        6759
+      ],
+      "marker": {
+        "color": "#1d4ed8"
+      },
+      "text": [
+        "13598",
+        "24366",
+        "6759"
+      ],
+      "textposition": "auto"
+    }
+  ],
+  "layout": {
+    "title": {
+      "text": "Complexity Overall Class Distribution"
+    },
+    "paper_bgcolor": "white",
+    "plot_bgcolor": "white",
+    "font": {
+      "family": "Arial, sans-serif",
+      "size": 12
+    },
+    "margin": {
+      "l": 60,
+      "r": 20,
+      "t": 60,
+      "b": 60
+    },
+    "xaxis": {
+      "title": {
+        "text": "Class"
+      },
+      "automargin": true
+    },
+    "yaxis": {
+      "title": {
+        "text": "Count"
+      },
+      "automargin": true
+    }
+  }
+}

dashboard/figures/cross-dataset-f1.plotly.json

@@ -0,0 +1,62 @@
+{
+  "data": [
+    {
+      "type": "bar",
+      "orientation": "v",
+      "x": [
+        "asap_aes",
+        "cambridge",
+        "cerf",
+        "commonlit",
+        "one_stop"
+      ],
+      "y": [
+        0.355652614304878,
+        0.5316323731138546,
+        0.6026179931249024,
+        0.33112328650236905,
+        0.26832434123074517
+      ],
+      "marker": {
+        "color": "#b45309"
+      },
+      "text": [
+        "0.356",
+        "0.532",
+        "0.603",
+        "0.331",
+        "0.268"
+      ],
+      "textposition": "auto"
+    }
+  ],
+  "layout": {
+    "title": {
+      "text": "Cross Dataset Evaluation"
+    },
+    "paper_bgcolor": "white",
+    "plot_bgcolor": "white",
+    "font": {
+      "family": "Arial, sans-serif",
+      "size": 12
+    },
+    "margin": {
+      "l": 60,
+      "r": 20,
+      "t": 60,
+      "b": 60
+    },
+    "xaxis": {
+      "title": {
+        "text": "Dataset"
+      },
+      "automargin": true
+    },
+    "yaxis": {
+      "title": {
+        "text": "F1 Macro"
+      },
+      "automargin": true
+    }
+  }
+}

dashboard/figures/eval-metrics.plotly.json

@@ -0,0 +1,65 @@
+{
+  "data": [
+    {
+      "type": "scatter",
+      "mode": "lines+markers",
+      "name": "Validation F1 Macro",
+      "x": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        6
+      ],
+      "y": [
+        0.6324,
+        0.655,
+        0.6605,
+        0.6473,
+        0.6302,
+        0.6283
+      ],
+      "line": {
+        "color": "#0f766e",
+        "width": 2
+      },
+      "marker": {
+        "size": 6
+      }
+    }
+  ],
+  "layout": {
+    "title": {
+      "text": "Evaluation Metrics"
+    },
+    "paper_bgcolor": "white",
+    "plot_bgcolor": "white",
+    "font": {
+      "family": "Arial, sans-serif",
+      "size": 12
+    },
+    "margin": {
+      "l": 60,
+      "r": 20,
+      "t": 60,
+      "b": 60
+    },
+    "xaxis": {
+      "title": {
+        "text": "Epoch"
+      },
+      "automargin": true
+    },
+    "yaxis": {
+      "title": {
+        "text": "Score"
+      },
+      "automargin": true
+    },
+    "legend": {
+      "orientation": "h",
+      "y": 1.12
+    }
+  }
+}

dashboard/figures/source-dataset-distribution.plotly.json

@@ -0,0 +1,53 @@
+{
+  "data": [
+    {
+      "type": "bar",
+      "orientation": "v",
+      "x": [
+        "asap-2-0",
+        "uk-key-stage-readability-for-english-texts"
+      ],
+      "y": [
+        24728,
+        19995
+      ],
+      "marker": {
+        "color": "#475569"
+      },
+      "text": [
+        "24728",
+        "19995"
+      ],
+      "textposition": "auto"
+    }
+  ],
+  "layout": {
+    "title": {
+      "text": "Complexity Overall Source Dataset Distribution"
+    },
+    "paper_bgcolor": "white",
+    "plot_bgcolor": "white",
+    "font": {
+      "family": "Arial, sans-serif",
+      "size": 12
+    },
+    "margin": {
+      "l": 60,
+      "r": 20,
+      "t": 60,
+      "b": 60
+    },
+    "xaxis": {
+      "title": {
+        "text": "Source Dataset"
+      },
+      "automargin": true
+    },
+    "yaxis": {
+      "title": {
+        "text": "Count"
+      },
+      "automargin": true
+    }
+  }
+}

dashboard/figures/training-loss.plotly.json

@@ -0,0 +1,93 @@
+{
+  "data": [
+    {
+      "type": "scatter",
+      "mode": "lines+markers",
+      "name": "Train Loss",
+      "x": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        6
+      ],
+      "y": [
+        0.8229,
+        0.488,
+        0.2493,
+        0.116,
+        0.0496,
+        0.0151
+      ],
+      "line": {
+        "color": "#0f766e",
+        "width": 2
+      },
+      "marker": {
+        "size": 6
+      }
+    },
+    {
+      "type": "scatter",
+      "mode": "lines+markers",
+      "name": "Validation Loss",
+      "x": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        6
+      ],
+      "y": [
+        0.7119,
+        0.6723,
+        0.8044,
+        1.2446,
+        1.5788,
+        1.9807
+      ],
+      "line": {
+        "color": "#b45309",
+        "width": 2
+      },
+      "marker": {
+        "size": 6
+      }
+    }
+  ],
+  "layout": {
+    "title": {
+      "text": "Training Loss"
+    },
+    "paper_bgcolor": "white",
+    "plot_bgcolor": "white",
+    "font": {
+      "family": "Arial, sans-serif",
+      "size": 12
+    },
+    "margin": {
+      "l": 60,
+      "r": 20,
+      "t": 60,
+      "b": 60
+    },
+    "xaxis": {
+      "title": {
+        "text": "Epoch"
+      },
+      "automargin": true
+    },
+    "yaxis": {
+      "title": {
+        "text": "Loss"
+      },
+      "automargin": true
+    },
+    "legend": {
+      "orientation": "h",
+      "y": 1.12
+    }
+  }
+}

dashboard/metadata/experiment-config.json

@@ -0,0 +1,75 @@
+{
+  "seed": 42,
+  "data_dir": "data",
+  "output_dir": "outputs",
+  "model_dir": "models",
+  "log_dir": "logs",
+  "task": "complexity-classification",
+  "text_col": "text",
+  "label_col": "label",
+  "num_labels": 3,
+  "encoded_label_col": "label_enc",
+  "train_pool_key": "train_pool",
+  "train_pool_datasets": [
+    "uk_key_stage",
+    "asap2"
+  ],
+  "external_eval_dataset_keys": [
+    "one_stop",
+    "commonlit",
+    "cerf",
+    "cambridge",
+    "asap_aes"
+  ],
+  "split_artifact_path": "data/complexity_splits.pkl",
+  "experiment_dataset_slug": "complexity",
+  "run_data_prep": false,
+  "run_transformer_eval": false,
+  "deberta_execution_mode": "legacy_eval",
+  "dl_train_mode": false,
+  "prefer_cached_internal_metrics": true,
+  "test_size": 0.15,
+  "val_size": 0.15,
+  "use_cv": true,
+  "cv_folds": 5,
+  "tfidf_word_ngrams": [
+    1,
+    2
+  ],
+  "use_char_ngrams": true,
+  "class_weight": "balanced",
+  "embedding_model": "fasttext-wiki-news-subwords-300",
+  "embedding_dim": 300,
+  "vocab_size": 50000,
+  "max_seq_len": 256,
+  "pad_token": "<PAD>",
+  "unk_token": "<UNK>",
+  "dl_batch_size": 64,
+  "dl_epochs": 20,
+  "dl_lr": 0.001,
+  "dl_patience": 3,
+  "dropout": 0.3,
+  "cnn_num_filters": 128,
+  "cnn_kernel_sizes": [
+    2,
+    3,
+    4
+  ],
+  "lstm_hidden_dim": 128,
+  "lstm_num_layers": 2,
+  "trf_model": "distilbert-base-uncased",
+  "max_len": 256,
+  "batch_size": 8,
+  "epochs": 3,
+  "lr": 2e-05,
+  "device": "mps",
+  "output_paths": {
+    "eda": "outputs/eda",
+    "metrics": "outputs/metrics",
+    "runs": "outputs/runs",
+    "plots_confusion": "outputs/plots/confusion_matrices",
+    "plots_comparison": "outputs/plots/comparisons",
+    "plots_learning": "outputs/plots/learning_curves",
+    "plots_history": "outputs/plots/histories"
+  }
+}

dashboard/metadata/model.json

@@ -0,0 +1,88 @@
+{
+  "model_id": "complexity-bilstm-final-v1",
+  "domain": "complexity",
+  "display_name": "Complexity BiLSTM Final v1",
+  "description": "Final BiLSTM with additive attention for CEFR-based text complexity classification.",
+  "is_active": true,
+  "priority": 5,
+  "framework": {
+    "type": "pytorch",
+    "task": "sequence-classification",
+    "library": "torch",
+    "architecture": "bilstm-attention",
+    "embeddings": "fasttext-wiki-news-subwords-300"
+  },
+  "artifacts": {
+    "weights": [
+      "models/glove_bilstm_complexity_train_pool_20260307_163358/best_model/best.pt"
+    ],
+    "tokenizer": [],
+    "config": [
+      "outputs/metrics/cfg_20260310_2125.json",
+      "models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_latest.json"
+    ],
+    "vocabulary": [
+      "models/shared/vocab.pkl"
+    ],
+    "label_map_file": "models/shared/class2id.json",
+    "label_classes_file": "models/shared/label_classes.json",
+    "label_encoder_file": "models/shared/label_encoder.pkl"
+  },
+  "model": {
+    "embedding_dim": 300,
+    "hidden_dim": 128,
+    "num_layers": 2,
+    "dropout": 0.3,
+    "bidirectional": true,
+    "attention": "additive",
+    "vocab_size": 50000,
+    "pad_token": "<PAD>",
+    "unk_token": "<UNK>"
+  },
+  "runtime": {
+    "max_sequence_length": 256,
+    "truncation": true,
+    "padding": true,
+    "batch_size": 1,
+    "device": "cpu",
+    "preprocessing": "normalize_text + preprocess_from_normalized + texts_to_sequences"
+  },
+  "inference": {
+    "return_probabilities": true,
+    "confidence_method": "softmax_max"
+  },
+  "ui": {
+    "domain_display_name": "Text Complexity",
+    "color_token": "complexity",
+    "group": "complexity-core"
+  },
+  "labels": {
+    "type": "single-label-classification",
+    "classes": [
+      {
+        "id": 0,
+        "name": "A1-A2",
+        "display_name": "A1-A2"
+      },
+      {
+        "id": 1,
+        "name": "B1-B2",
+        "display_name": "B1-B2"
+      },
+      {
+        "id": 2,
+        "name": "C1-C2",
+        "display_name": "C1-C2"
+      }
+    ]
+  },
+  "prod_model_dir": "app/app-models/prod-model-complexity",
+  "artifacts_present": [
+    "app/app-models/prod-model-complexity/.DS_Store",
+    "app/app-models/prod-model-complexity/best.pt",
+    "app/app-models/prod-model-complexity/label_classes.json",
+    "app/app-models/prod-model-complexity/model-config.yaml",
+    "app/app-models/prod-model-complexity/test_inference.ipynb",
+    "app/app-models/prod-model-complexity/vocab.pkl"
+  ]
+}

dashboard/metrics/benchmark-test.json

@@ -0,0 +1,122 @@
+[
+  {
+    "model": "DeBERTaV3_train_pool",
+    "display_name": "DeBERTaV3 train pool",
+    "family": "transformer",
+    "split": "test",
+    "accuracy": 0.7791026978685348,
+    "f1_macro": 0.7510812492382728,
+    "f1_weighted": 0.7760556709086348,
+    "loss": 0.5336738107592773,
+    "source_file": "complexity/outputs/metrics/results_complexity-classification.csv",
+    "is_production": false
+  },
+  {
+    "model": "DistilBERT_train_pool",
+    "display_name": "DistilBERT train pool",
+    "family": "transformer",
+    "split": "test",
+    "accuracy": 0.7746310925622298,
+    "f1_macro": 0.7430667400906253,
+    "f1_weighted": 0.7720896637566615,
+    "loss": null,
+    "source_file": "complexity/outputs/metrics/results_complexity-classification.csv",
+    "is_production": false
+  },
+  {
+    "model": "RoBERTa_train_pool",
+    "display_name": "RoBERTa train pool",
+    "family": "transformer",
+    "split": "test",
+    "accuracy": 0.7676255775823521,
+    "f1_macro": 0.7297663021875974,
+    "f1_weighted": 0.7644717465447768,
+    "loss": null,
+    "source_file": "complexity/outputs/metrics/results_complexity-classification.csv",
+    "is_production": false
+  },
+  {
+    "model": "TF-IDF + SVC [train_pool]",
+    "display_name": "TF-IDF + SVC [train pool]",
+    "family": "classical",
+    "split": "test",
+    "accuracy": 0.7002533909673573,
+    "f1_macro": 0.6686789738725382,
+    "f1_weighted": 0.699786931830328,
+    "loss": null,
+    "source_file": "complexity/outputs/metrics/results_complexity-classification.csv",
+    "is_production": false
+  },
+  {
+    "model": "TF-IDF + LogReg [train_pool]",
+    "display_name": "TF-IDF + LogReg [train pool]",
+    "family": "classical",
+    "split": "test",
+    "accuracy": 0.6835593978238188,
+    "f1_macro": 0.6607874127954448,
+    "f1_weighted": 0.6863970572448843,
+    "loss": null,
+    "source_file": "complexity/outputs/metrics/results_complexity-classification.csv",
+    "is_production": false
+  },
+  {
+    "model": "GloVe BiLSTM [train_pool]",
+    "display_name": "GloVe BiLSTM [train pool]",
+    "family": "deep",
+    "split": "test",
+    "accuracy": 0.6926516619466389,
+    "f1_macro": 0.6551661294897166,
+    "f1_weighted": 0.6923209903034657,
+    "loss": null,
+    "source_file": "complexity/outputs/metrics/results_complexity-classification.csv",
+    "is_production": true
+  },
+  {
+    "model": "GloVe CNN [train_pool]",
+    "display_name": "GloVe CNN [train pool]",
+    "family": "deep",
+    "split": "test",
+    "accuracy": 0.6795349530481443,
+    "f1_macro": 0.6388266463257223,
+    "f1_weighted": 0.6766999349479854,
+    "loss": null,
+    "source_file": "complexity/outputs/metrics/results_complexity-classification.csv",
+    "is_production": false
+  },
+  {
+    "model": "GloVe MLP [train_pool]",
+    "display_name": "GloVe MLP [train pool]",
+    "family": "deep",
+    "split": "test",
+    "accuracy": 0.6527053212103145,
+    "f1_macro": 0.5964460940300637,
+    "f1_weighted": 0.6426749637907794,
+    "loss": null,
+    "source_file": "complexity/outputs/metrics/results_complexity-classification.csv",
+    "is_production": false
+  },
+  {
+    "model": "TF-IDF + XGBoost [train_pool]",
+    "display_name": "TF-IDF + XGBoost [train pool]",
+    "family": "classical",
+    "split": "test",
+    "accuracy": 0.673870919660158,
+    "f1_macro": 0.534220517963146,
+    "f1_weighted": 0.6343891751219696,
+    "loss": null,
+    "source_file": "complexity/outputs/metrics/results_complexity-classification.csv",
+    "is_production": false
+  },
+  {
+    "model": "Majority Class [train_pool]",
+    "display_name": "Majority Class [train pool]",
+    "family": "classical",
+    "split": "test",
+    "accuracy": 0.5447905798181547,
+    "f1_macro": 0.2351087096359192,
+    "f1_weighted": 0.3842550307285518,
+    "loss": null,
+    "source_file": "complexity/outputs/metrics/results_complexity-classification.csv",
+    "is_production": false
+  }
+]

dashboard/metrics/cross-dataset.json

@@ -0,0 +1,47 @@
+[
+  {
+    "family": "deep",
+    "dataset": "asap_aes",
+    "model": "GloVe BiLSTM",
+    "accuracy": 0.414122725871107,
+    "f1_macro": 0.355652614304878,
+    "f1_weighted": 0.37092120365396425,
+    "source_file": "complexity/outputs/cross_dataset_eval/summary_latest.csv"
+  },
+  {
+    "family": "deep",
+    "dataset": "cambridge",
+    "model": "GloVe BiLSTM",
+    "accuracy": 0.5437956204379562,
+    "f1_macro": 0.5316323731138546,
+    "f1_weighted": 0.5240329218106996,
+    "source_file": "complexity/outputs/cross_dataset_eval/summary_latest.csv"
+  },
+  {
+    "family": "deep",
+    "dataset": "cerf",
+    "model": "GloVe BiLSTM",
+    "accuracy": 0.5934360348292029,
+    "f1_macro": 0.6026179931249024,
+    "f1_weighted": 0.6005301829938762,
+    "source_file": "complexity/outputs/cross_dataset_eval/summary_latest.csv"
+  },
+  {
+    "family": "deep",
+    "dataset": "commonlit",
+    "model": "GloVe BiLSTM",
+    "accuracy": 0.4262121532923989,
+    "f1_macro": 0.33112328650236905,
+    "f1_weighted": 0.3841412546822796,
+    "source_file": "complexity/outputs/cross_dataset_eval/summary_latest.csv"
+  },
+  {
+    "family": "deep",
+    "dataset": "one_stop",
+    "model": "GloVe BiLSTM",
+    "accuracy": 0.34714003944773175,
+    "f1_macro": 0.26832434123074517,
+    "f1_weighted": 0.26092763705495803,
+    "source_file": "complexity/outputs/cross_dataset_eval/summary_latest.csv"
+  }
+]

dashboard/metrics/primary-evaluation.json

@@ -0,0 +1,22 @@
+{
+  "model": "GloVe BiLSTM [train_pool]",
+  "source_files": [
+    "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_latest.json"
+  ],
+  "splits": {
+    "val": {
+      "accuracy": 0.682963183782978,
+      "f1_macro": 0.6450992102932628,
+      "f1_weighted": 0.6830474215229027
+    },
+    "test": {
+      "accuracy": 0.6926516619466389,
+      "f1_macro": 0.6551661294897166,
+      "f1_weighted": 0.6923209903034657
+    }
+  },
+  "artifact_paths": {
+    "experiment_dir": "models/glove_bilstm_complexity_train_pool_20260307_163358",
+    "best_checkpoint": "models/glove_bilstm_complexity_train_pool_20260307_163358/best_model/best.pt"
+  }
+}

dashboard/samples/prediction-samples.json

@@ -0,0 +1,114 @@
+[
+  {
+    "example_id": "1",
+    "text": "The boy has a small dog. The dog is brown and friendly.",
+    "production_prediction": {
+      "model": "GloVe BiLSTM",
+      "label": "B1-B2",
+      "confidence": 0.9545841813087463
+    },
+    "reference_prediction": {
+      "model": "DeBERTaV3",
+      "label": "B1-B2",
+      "confidence": 0.5645282864570618
+    }
+  },
+  {
+    "example_id": "2",
+    "text": "My sister goes to school by bus and comes home in the afternoon.",
+    "production_prediction": {
+      "model": "GloVe BiLSTM",
+      "label": "A1-A2",
+      "confidence": 0.6837078332901001
+    },
+    "reference_prediction": {
+      "model": "DeBERTaV3",
+      "label": "B1-B2",
+      "confidence": 0.5060160160064697
+    }
+  },
+  {
+    "example_id": "3",
+    "text": "The museum opened a new room where visitors can learn about life in ancient cities.",
+    "production_prediction": {
+      "model": "GloVe BiLSTM",
+      "label": "B1-B2",
+      "confidence": 0.521766185760498
+    },
+    "reference_prediction": {
+      "model": "DeBERTaV3",
+      "label": "B1-B2",
+      "confidence": 0.5297347903251648
+    }
+  },
+  {
+    "example_id": "4",
+    "text": "Although the weather forecast predicted heavy rain, the match continued because the pitch remained playable.",
+    "production_prediction": {
+      "model": "GloVe BiLSTM",
+      "label": "B1-B2",
+      "confidence": 0.6220280528068542
+    },
+    "reference_prediction": {
+      "model": "DeBERTaV3",
+      "label": "B1-B2",
+      "confidence": 0.4810847342014313
+    }
+  },
+  {
+    "example_id": "5",
+    "text": "Researchers compared the two groups carefully in order to determine whether the intervention had any measurable effect.",
+    "production_prediction": {
+      "model": "GloVe BiLSTM",
+      "label": "B1-B2",
+      "confidence": 0.39631497859954834
+    },
+    "reference_prediction": {
+      "model": "DeBERTaV3",
+      "label": "B1-B2",
+      "confidence": 0.4548090994358063
+    }
+  },
+  {
+    "example_id": "6",
+    "text": "Notwithstanding the apparent simplicity of the proposal, its implementation would require a sequence of interdependent regulatory revisions.",
+    "production_prediction": {
+      "model": "GloVe BiLSTM",
+      "label": "C1-C2",
+      "confidence": 0.9676215052604675
+    },
+    "reference_prediction": {
+      "model": "DeBERTaV3",
+      "label": "B1-B2",
+      "confidence": 0.4681660234928131
+    }
+  },
+  {
+    "example_id": "7",
+    "text": "The committee acknowledged that, in the absence of reliable longitudinal evidence, any definitive conclusion would remain provisional.",
+    "production_prediction": {
+      "model": "GloVe BiLSTM",
+      "label": "A1-A2",
+      "confidence": 0.4590006172657013
+    },
+    "reference_prediction": {
+      "model": "DeBERTaV3",
+      "label": "C1-C2",
+      "confidence": 0.4668876528739929
+    }
+  },
+  {
+    "example_id": "8",
+    "text": "By foregrounding the tension between economic efficiency and social equity, the report complicates any straightforward interpretation of the policy debate.",
+    "production_prediction": {
+      "model": "GloVe BiLSTM",
+      "label": "B1-B2",
+      "confidence": 0.3824934661388397
+    },
+    "reference_prediction": {
+      "model": "DeBERTaV3",
+      "label": "B1-B2",
+      "confidence": 0.4470742642879486
+    }
+  }
+]

dashboard/summary/overview.json

@@ -0,0 +1,53 @@
+{
+  "generated_at": "2026-03-12T17:02:22+00:00",
+  "model": {
+    "model_id": "complexity-bilstm-final-v1",
+    "display_name": "Complexity BiLSTM Final v1",
+    "domain": "complexity",
+    "framework_type": "pytorch",
+    "framework_architecture": "bilstm-attention"
+  },
+  "evaluation_highlights": {
+    "validation": {
+      "accuracy": 0.682963183782978,
+      "f1_macro": 0.6450992102932628,
+      "f1_weighted": 0.6830474215229027
+    },
+    "test": {
+      "accuracy": 0.6926516619466389,
+      "f1_macro": 0.6551661294897166,
+      "f1_weighted": 0.6923209903034657
+    },
+    "benchmark_rank": {
+      "rank": 6,
+      "out_of": 10,
+      "metric": "f1_macro",
+      "score": 0.6551661294897166
+    }
+  },
+  "cross_dataset_highlights": {
+    "datasets": 5,
+    "mean_accuracy": 0.464941,
+    "mean_f1_macro": 0.41787,
+    "mean_f1_weighted": 0.428111
+  },
+  "artifact_status": {
+    "metadata": "available",
+    "evaluation": "available",
+    "benchmark": "available",
+    "training_curves": "available",
+    "learning_curves": "not_applicable",
+    "cross_dataset": "available",
+    "confusion_matrix": "image_only",
+    "class_distribution": "missing",
+    "samples": "available"
+  },
+  "notes": [
+    "Primary evaluation uses the production GloVe BiLSTM metrics_latest.json referenced by prod-model-complexity/model-config.yaml.",
+    "Complexity is the richest dashboard domain in the repo: internal benchmark data, external evaluation summaries, split parquets, manual sample predictions, and prod-model training logs are all present.",
+    "Confusion matrices exist only as saved images for the production model; no numeric matrix artifact was found.",
+    "class_distribution parsing failed for complexity/data/train_pool_train.parquet: Parquet dashboard sources require the optional pandas dependency.",
+    "class_distribution parsing failed for complexity/data/train_pool_val.parquet: Parquet dashboard sources require the optional pandas dependency.",
+    "class_distribution parsing failed for complexity/data/train_pool_test.parquet: Parquet dashboard sources require the optional pandas dependency."
+  ]
+}

dashboard/summary/source-audit.json

@@ -0,0 +1,1218 @@
+{
+  "generated_at": "2026-03-12T17:02:22+00:00",
+  "scanned_roots": [
+    "app/app-models/prod-model-complexity",
+    "complexity/data",
+    "complexity/logs",
+    "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs",
+    "complexity/outputs/cross_dataset_eval",
+    "complexity/outputs/metrics",
+    "complexity/outputs/plots/confusion_matrices"
+  ],
+  "artifact_counts": {
+    "logs": 815,
+    "metrics_and_configs": 146,
+    "other_structured_files": 40,
+    "plot_images": 24,
+    "structured_data": 11,
+    "tabular_data": 82
+  },
+  "candidate_artifacts_by_category": {
+    "logs": [
+      "complexity/data/archive/asap-2-0/source.txt",
+      "complexity/data/archive/asap-aes/source.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/1.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/10.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/11.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/12.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/13.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/14.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/15.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/16.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/17.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/18.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/19.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/2.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/20.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/21.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/22.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/23.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/24.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/25.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/26.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/27.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/28.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/29.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/3.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/30.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/31.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/32.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/33.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/34.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/35.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/36.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/37.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/38.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/39.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/4.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/40.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/41.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/42.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/43.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/44.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/45.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/46.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/47.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/48.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/49.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/5.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/50.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/51.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/52.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/53.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/54.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/55.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/6.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/7.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/8.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE/9.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE.txt",
+      "complexity/data/archive/cambridge-english-readability/CAE_raw.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/1.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/10.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/11.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/12.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/13.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/14.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/15.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/16.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/17.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/18.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/19.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/2.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/20.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/21.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/22.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/23.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/24.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/25.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/26.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/27.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/28.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/29.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/3.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/30.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/31.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/32.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/33.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/34.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/35.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/36.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/37.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/38.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/39.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/4.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/40.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/41.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/42.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/43.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/44.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/5.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/6.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/7.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/8.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE/9.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE.txt",
+      "complexity/data/archive/cambridge-english-readability/CPE_raw.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/1.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/10.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/11.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/12.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/13.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/14.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/15.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/16.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/17.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/18.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/19.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/2.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/20.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/21.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/22.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/23.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/24.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/25.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/26.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/27.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/28.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/29.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/3.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/30.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/31.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/32.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/33.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/34.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/35.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/36.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/37.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/38.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/39.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/4.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/40.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/41.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/42.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/43.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/44.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/45.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/46.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/47.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/48.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/49.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/5.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/50.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/51.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/52.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/53.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/54.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/55.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/56.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/57.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/58.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/59.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/6.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/60.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/61.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/62.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/63.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/64.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/65.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/66.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/67.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/7.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/8.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE/9.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE.txt",
+      "complexity/data/archive/cambridge-english-readability/FCE_raw.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/1.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/10.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/11.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/12.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/13.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/14.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/15.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/16.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/17.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/18.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/19.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/2.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/20.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/21.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/22.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/23.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/24.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/25.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/26.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/27.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/28.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/29.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/3.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/30.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/31.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/32.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/33.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/34.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/35.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/36.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/37.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/38.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/39.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/4.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/40.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/41.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/42.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/43.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/44.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/45.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/46.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/47.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/48.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/49.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/5.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/50.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/51.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/52.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/53.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/54.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/55.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/56.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/57.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/58.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/59.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/6.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/60.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/61.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/7.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/8.txt",
+      "complexity/data/archive/cambridge-english-readability/KET/9.txt",
+      "complexity/data/archive/cambridge-english-readability/KET.txt",
+      "complexity/data/archive/cambridge-english-readability/KET_raw.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/1.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/10.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/11.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/12.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/13.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/14.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/15.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/16.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/17.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/18.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/19.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/2.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/20.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/21.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/22.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/23.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/24.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/25.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/26.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/27.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/28.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/29.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/3.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/30.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/31.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/32.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/33.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/34.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/35.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/36.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/37.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/38.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/39.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/4.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/40.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/41.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/42.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/43.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/44.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/45.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/46.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/47.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/48.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/49.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/5.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/50.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/51.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/52.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/53.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/54.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/55.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/6.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/7.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/8.txt",
+      "complexity/data/archive/cambridge-english-readability/PET/9.txt",
+      "complexity/data/archive/cambridge-english-readability/PET.txt",
+      "complexity/data/archive/cambridge-english-readability/PET_raw.txt",
+      "complexity/data/archive/cambridge-english-readability/source.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/1.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/10.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/100.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/101.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/102.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/103.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/104.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/105.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/106.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/107.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/108.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/109.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/11.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/110.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/111.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/112.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/113.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/114.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/115.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/116.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/117.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/118.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/119.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/12.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/120.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/121.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/122.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/123.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/124.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/125.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/126.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/127.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/128.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/129.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/13.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/130.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/131.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/132.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/133.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/134.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/135.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/136.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/137.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/138.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/139.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/14.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/140.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/141.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/142.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/143.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/144.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/145.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/146.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/147.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/148.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/149.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/15.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/150.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/151.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/152.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/153.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/154.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/155.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/156.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/157.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/158.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/159.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/16.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/160.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/161.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/17.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/18.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/19.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/2.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/20.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/21.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/22.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/23.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/24.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/25.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/26.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/27.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/28.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/29.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/3.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/30.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/31.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/32.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/33.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/34.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/35.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/36.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/37.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/38.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/39.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/4.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/40.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/41.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/42.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/43.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/44.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/45.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/46.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/47.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/48.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/49.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/5.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/50.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/51.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/52.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/53.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/54.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/55.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/56.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/57.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/58.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/59.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/6.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/60.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/61.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/62.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/63.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/64.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/65.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/66.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/67.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/68.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/69.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/7.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/70.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/71.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/72.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/73.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/74.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/75.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/76.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/77.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/78.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/79.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/8.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/80.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/81.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/82.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/83.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/84.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/85.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/86.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/87.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/88.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/89.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/9.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/90.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/91.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/92.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/93.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/94.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/95.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/96.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/97.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/98.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv/99.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv.txt",
+      "complexity/data/archive/one-stop-corpus-english/Adv_raw.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/1.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/10.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/100.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/101.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/102.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/103.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/104.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/105.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/106.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/107.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/108.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/109.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/11.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/110.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/111.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/112.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/113.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/114.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/115.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/116.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/117.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/118.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/119.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/12.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/120.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/121.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/122.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/123.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/124.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/125.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/126.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/127.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/128.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/129.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/13.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/130.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/131.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/132.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/133.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/134.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/135.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/136.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/137.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/138.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/139.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/14.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/140.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/141.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/142.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/143.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/144.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/145.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/146.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/147.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/148.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/149.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/15.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/150.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/151.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/152.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/153.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/154.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/155.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/156.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/157.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/158.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/159.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/16.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/160.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/161.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/162.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/163.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/164.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/165.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/166.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/167.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/168.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/169.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/17.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/170.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/171.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/172.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/173.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/174.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/175.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/176.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/177.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/178.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/18.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/19.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/2.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/20.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/21.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/22.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/23.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/24.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/25.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/26.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/27.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/28.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/29.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/3.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/30.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/31.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/32.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/33.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/34.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/35.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/36.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/37.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/38.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/39.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/4.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/40.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/41.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/42.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/43.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/44.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/45.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/46.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/47.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/48.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/49.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/5.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/50.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/51.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/52.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/53.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/54.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/55.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/56.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/57.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/58.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/59.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/6.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/60.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/61.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/62.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/63.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/64.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/65.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/66.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/67.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/68.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/69.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/7.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/70.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/71.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/72.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/73.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/74.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/75.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/76.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/77.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/78.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/79.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/8.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/80.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/81.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/82.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/83.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/84.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/85.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/86.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/87.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/88.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/89.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/9.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/90.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/91.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/92.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/93.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/94.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/95.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/96.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/97.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/98.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele/99.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele.txt",
+      "complexity/data/archive/one-stop-corpus-english/Ele_raw.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/1.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/10.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/100.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/101.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/102.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/103.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/104.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/105.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/106.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/107.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/108.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/109.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/11.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/110.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/111.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/112.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/113.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/114.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/115.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/116.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/117.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/118.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/119.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/12.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/120.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/121.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/122.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/123.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/124.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/125.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/126.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/127.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/128.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/129.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/13.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/130.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/131.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/132.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/133.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/134.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/135.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/136.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/137.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/138.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/139.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/14.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/140.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/141.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/142.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/143.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/144.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/145.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/146.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/147.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/148.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/149.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/15.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/150.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/151.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/152.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/153.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/154.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/155.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/156.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/157.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/158.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/159.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/16.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/160.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/161.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/162.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/163.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/164.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/165.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/166.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/167.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/168.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/169.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/17.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/170.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/18.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/19.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/2.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/20.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/21.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/22.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/23.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/24.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/25.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/26.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/27.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/28.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/29.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/3.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/30.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/31.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/32.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/33.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/34.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/35.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/36.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/37.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/38.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/39.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/4.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/40.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/41.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/42.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/43.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/44.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/45.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/46.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/47.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/48.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/49.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/5.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/50.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/51.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/52.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/53.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/54.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/55.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/56.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/57.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/58.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/59.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/6.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/60.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/61.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/62.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/63.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/64.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/65.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/66.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/67.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/68.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/69.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/7.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/70.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/71.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/72.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/73.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/74.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/75.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/76.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/77.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/78.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/79.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/8.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/80.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/81.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/82.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/83.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/84.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/85.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/86.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/87.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/88.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/89.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/9.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/90.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/91.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/92.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/93.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/94.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/95.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/96.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/97.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/98.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int/99.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int.txt",
+      "complexity/data/archive/one-stop-corpus-english/Int_raw.txt",
+      "complexity/data/archive/uk-key-stage-readability-for-english-texts/source.txt",
+      "complexity/data/archive/\u0441ommonlit-readability-prize/source.txt",
+      "complexity/logs/complexity-classification_GloVe_BiLSTM_[train_pool].log",
+      "complexity/logs/complexity-classification_GloVe_CNN_[train_pool].log",
+      "complexity/logs/complexity-classification_GloVe_MLP_[train_pool].log"
+    ],
+    "metrics_and_configs": [
+      "app/app-models/prod-model-complexity/dashboard/figures/eval-metrics.plotly.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260307_164217.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260307_202849.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260308_105127.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260308_133743.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260308_140555.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260309_151810.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260310_122220.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260310_122755.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260310_133023.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260310_135252.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260310_180434.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_20260310_182313.json",
+      "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/majority_class/metrics_20260307_204030.json",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/majority_class/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/tf_idf_logreg/metrics_20260307_204016.json",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/tf_idf_logreg/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/tf_idf_svc/metrics_20260307_204010.json",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/tf_idf_svc/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/tf_idf_xgboost/metrics_20260307_204024.json",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/tf_idf_xgboost/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/majority_class/metrics_20260307_204004.json",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/majority_class/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/tf_idf_logreg/metrics_20260307_204003.json",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/tf_idf_logreg/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/tf_idf_svc/metrics_20260307_204003.json",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/tf_idf_svc/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/tf_idf_xgboost/metrics_20260307_204003.json",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/tf_idf_xgboost/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/majority_class/metrics_20260307_204002.json",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/majority_class/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/tf_idf_logreg/metrics_20260307_203959.json",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/tf_idf_logreg/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/tf_idf_svc/metrics_20260307_203958.json",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/tf_idf_svc/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/tf_idf_xgboost/metrics_20260307_204001.json",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/tf_idf_xgboost/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/majority_class/metrics_20260307_203957.json",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/majority_class/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/tf_idf_logreg/metrics_20260307_203953.json",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/tf_idf_logreg/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/tf_idf_svc/metrics_20260307_203951.json",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/tf_idf_svc/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/tf_idf_xgboost/metrics_20260307_203955.json",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/tf_idf_xgboost/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/majority_class/metrics_20260307_203949.json",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/majority_class/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/tf_idf_logreg/metrics_20260307_203947.json",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/tf_idf_logreg/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/tf_idf_svc/metrics_20260307_203946.json",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/tf_idf_svc/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/tf_idf_xgboost/metrics_20260307_203948.json",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/tf_idf_xgboost/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/asap_aes/glove_bilstm/metrics_20260307_204056.json",
+      "complexity/outputs/cross_dataset_eval/deep/asap_aes/glove_bilstm/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/asap_aes/glove_cnn/metrics_20260307_204048.json",
+      "complexity/outputs/cross_dataset_eval/deep/asap_aes/glove_cnn/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/asap_aes/glove_mlp/metrics_20260307_204046.json",
+      "complexity/outputs/cross_dataset_eval/deep/asap_aes/glove_mlp/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/cambridge/glove_bilstm/metrics_20260307_204040.json",
+      "complexity/outputs/cross_dataset_eval/deep/cambridge/glove_bilstm/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/cambridge/glove_cnn/metrics_20260307_204040.json",
+      "complexity/outputs/cross_dataset_eval/deep/cambridge/glove_cnn/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/cambridge/glove_mlp/metrics_20260307_204039.json",
+      "complexity/outputs/cross_dataset_eval/deep/cambridge/glove_mlp/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/cerf/glove_bilstm/metrics_20260307_204039.json",
+      "complexity/outputs/cross_dataset_eval/deep/cerf/glove_bilstm/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/cerf/glove_cnn/metrics_20260307_204038.json",
+      "complexity/outputs/cross_dataset_eval/deep/cerf/glove_cnn/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/cerf/glove_mlp/metrics_20260307_204038.json",
+      "complexity/outputs/cross_dataset_eval/deep/cerf/glove_mlp/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/commonlit/glove_bilstm/metrics_20260307_204036.json",
+      "complexity/outputs/cross_dataset_eval/deep/commonlit/glove_bilstm/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/commonlit/glove_cnn/metrics_20260307_204034.json",
+      "complexity/outputs/cross_dataset_eval/deep/commonlit/glove_cnn/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/commonlit/glove_mlp/metrics_20260307_204033.json",
+      "complexity/outputs/cross_dataset_eval/deep/commonlit/glove_mlp/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/one_stop/glove_bilstm/metrics_20260307_204031.json",
+      "complexity/outputs/cross_dataset_eval/deep/one_stop/glove_bilstm/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/one_stop/glove_cnn/metrics_20260307_204031.json",
+      "complexity/outputs/cross_dataset_eval/deep/one_stop/glove_cnn/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/deep/one_stop/glove_mlp/metrics_20260307_204031.json",
+      "complexity/outputs/cross_dataset_eval/deep/one_stop/glove_mlp/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/asap_aes/metrics_20260307_205123.json",
+      "complexity/outputs/cross_dataset_eval/transformer/asap_aes/metrics_20260308_115229.json",
+      "complexity/outputs/cross_dataset_eval/transformer/asap_aes/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/cambridge/metrics_20260307_204929.json",
+      "complexity/outputs/cross_dataset_eval/transformer/cambridge/metrics_20260308_114603.json",
+      "complexity/outputs/cross_dataset_eval/transformer/cambridge/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/cerf/metrics_20260307_204925.json",
+      "complexity/outputs/cross_dataset_eval/transformer/cerf/metrics_20260308_114553.json",
+      "complexity/outputs/cross_dataset_eval/transformer/cerf/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/commonlit/metrics_20260307_204911.json",
+      "complexity/outputs/cross_dataset_eval/transformer/commonlit/metrics_20260308_114507.json",
+      "complexity/outputs/cross_dataset_eval/transformer/commonlit/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/asap_aes/metrics_20260308_122313.json",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/asap_aes/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/cambridge/metrics_20260308_121646.json",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/cambridge/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/cerf/metrics_20260308_121637.json",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/cerf/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/commonlit/metrics_20260308_121550.json",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/commonlit/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/one_stop/metrics_20260308_121328.json",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/one_stop/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/asap_aes/metrics_20260308_122612.json",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/asap_aes/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/cambridge/metrics_20260308_122418.json",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/cambridge/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/cerf/metrics_20260308_122415.json",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/cerf/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/commonlit/metrics_20260308_122401.json",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/commonlit/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/one_stop/metrics_20260308_122318.json",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/one_stop/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/one_stop/metrics_20260307_204829.json",
+      "complexity/outputs/cross_dataset_eval/transformer/one_stop/metrics_20260308_114246.json",
+      "complexity/outputs/cross_dataset_eval/transformer/one_stop/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/asap_aes/metrics_20260308_123146.json",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/asap_aes/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/cambridge/metrics_20260308_122812.json",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/cambridge/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/cerf/metrics_20260308_122806.json",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/cerf/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/commonlit/metrics_20260308_122740.json",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/commonlit/metrics_latest.json",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/one_stop/metrics_20260308_122621.json",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/one_stop/metrics_latest.json",
+      "complexity/outputs/metrics/results_20260308_1242.json",
+      "complexity/outputs/metrics/results_20260308_1406.json",
+      "complexity/outputs/metrics/results_20260310_1225.json",
+      "complexity/outputs/metrics/results_20260310_1228.json",
+      "complexity/outputs/metrics/results_20260310_1331.json",
+      "complexity/outputs/metrics/results_20260310_1353.json",
+      "complexity/outputs/metrics/results_20260310_1805.json",
+      "complexity/outputs/metrics/results_20260310_1824.json",
+      "complexity/outputs/metrics/results_20260310_2039.json",
+      "complexity/outputs/metrics/results_20260310_2112.json",
+      "complexity/outputs/metrics/results_20260310_2125.json",
+      "complexity/outputs/metrics/results_20260310_2213.json",
+      "complexity/outputs/metrics/results_20260310_2229.json",
+      "complexity/outputs/metrics/results_20260311_1746.json",
+      "complexity/outputs/metrics/results_20260311_1802.json",
+      "complexity/outputs/metrics/results_20260311_1816.json",
+      "complexity/outputs/metrics/results_20260311_1836.json"
+    ],
+    "other_structured_files": [
+      "app/app-models/prod-model-complexity/dashboard/curves/training-history.json",
+      "app/app-models/prod-model-complexity/dashboard/dashboard-manifest.json",
+      "app/app-models/prod-model-complexity/dashboard/distributions/class-distribution.json",
+      "app/app-models/prod-model-complexity/dashboard/distributions/source-dataset-distribution.json",
+      "app/app-models/prod-model-complexity/dashboard/figures/benchmark-test-f1.plotly.json",
+      "app/app-models/prod-model-complexity/dashboard/figures/class-distribution.plotly.json",
+      "app/app-models/prod-model-complexity/dashboard/figures/cross-dataset-f1.plotly.json",
+      "app/app-models/prod-model-complexity/dashboard/figures/source-dataset-distribution.plotly.json",
+      "app/app-models/prod-model-complexity/dashboard/figures/training-loss.plotly.json",
+      "app/app-models/prod-model-complexity/dashboard/metadata/experiment-config.json",
+      "app/app-models/prod-model-complexity/dashboard/metadata/model.json",
+      "app/app-models/prod-model-complexity/dashboard/metrics/benchmark-test.json",
+      "app/app-models/prod-model-complexity/dashboard/metrics/cross-dataset.json",
+      "app/app-models/prod-model-complexity/dashboard/metrics/primary-evaluation.json",
+      "app/app-models/prod-model-complexity/dashboard/samples/prediction-samples.json",
+      "app/app-models/prod-model-complexity/dashboard/summary/overview.json",
+      "app/app-models/prod-model-complexity/dashboard/summary/source-audit.json",
+      "app/app-models/prod-model-complexity/label_classes.json",
+      "app/app-models/prod-model-complexity/model-config.yaml",
+      "complexity/outputs/metrics/baseline_model_registry.json",
+      "complexity/outputs/metrics/cfg_20260308_1242.json",
+      "complexity/outputs/metrics/cfg_20260308_1406.json",
+      "complexity/outputs/metrics/cfg_20260310_1225.json",
+      "complexity/outputs/metrics/cfg_20260310_1228.json",
+      "complexity/outputs/metrics/cfg_20260310_1331.json",
+      "complexity/outputs/metrics/cfg_20260310_1353.json",
+      "complexity/outputs/metrics/cfg_20260310_1805.json",
+      "complexity/outputs/metrics/cfg_20260310_1824.json",
+      "complexity/outputs/metrics/cfg_20260310_2039.json",
+      "complexity/outputs/metrics/cfg_20260310_2112.json",
+      "complexity/outputs/metrics/cfg_20260310_2125.json",
+      "complexity/outputs/metrics/cfg_20260310_2213.json",
+      "complexity/outputs/metrics/cfg_20260310_2229.json",
+      "complexity/outputs/metrics/cfg_20260311_1746.json",
+      "complexity/outputs/metrics/cfg_20260311_1802.json",
+      "complexity/outputs/metrics/cfg_20260311_1816.json",
+      "complexity/outputs/metrics/cfg_20260311_1836.json",
+      "complexity/outputs/metrics/transformers/deberta_v3_base_train_pool_latest.json",
+      "complexity/outputs/metrics/transformers/distilbert_train_pool_latest.json",
+      "complexity/outputs/metrics/transformers/roberta_train_pool_latest.json"
+    ],
+    "plot_images": [
+      "app/app-models/prod-model-complexity/dashboard/confusion/cm-glove-bilstm-train-pool.png",
+      "app/app-models/prod-model-complexity/dashboard/confusion/internal-test-confusion-matrix.png",
+      "complexity/outputs/cross_dataset_eval/plots/cross_dataset_overview_f1_macro.png",
+      "complexity/outputs/plots/confusion_matrices/cm_debertav3_train_pool.png",
+      "complexity/outputs/plots/confusion_matrices/cm_debertav3_train_pool_asap_aes.png",
+      "complexity/outputs/plots/confusion_matrices/cm_debertav3_train_pool_cambridge.png",
+      "complexity/outputs/plots/confusion_matrices/cm_debertav3_train_pool_cerf.png",
+      "complexity/outputs/plots/confusion_matrices/cm_debertav3_train_pool_commonlit.png",
+      "complexity/outputs/plots/confusion_matrices/cm_debertav3_train_pool_one_stop.png",
+      "complexity/outputs/plots/confusion_matrices/cm_distilbert_train_pool.png",
+      "complexity/outputs/plots/confusion_matrices/cm_distilbert_train_pool_asap_aes.png",
+      "complexity/outputs/plots/confusion_matrices/cm_distilbert_train_pool_cambridge.png",
+      "complexity/outputs/plots/confusion_matrices/cm_distilbert_train_pool_cerf.png",
+      "complexity/outputs/plots/confusion_matrices/cm_distilbert_train_pool_commonlit.png",
+      "complexity/outputs/plots/confusion_matrices/cm_distilbert_train_pool_one_stop.png",
+      "complexity/outputs/plots/confusion_matrices/cm_glove_bilstm_train_pool.png",
+      "complexity/outputs/plots/confusion_matrices/cm_glove_cnn_train_pool.png",
+      "complexity/outputs/plots/confusion_matrices/cm_glove_mlp_train_pool.png",
+      "complexity/outputs/plots/confusion_matrices/cm_roberta_train_pool.png",
+      "complexity/outputs/plots/confusion_matrices/cm_roberta_train_pool_asap_aes.png",
+      "complexity/outputs/plots/confusion_matrices/cm_roberta_train_pool_cambridge.png",
+      "complexity/outputs/plots/confusion_matrices/cm_roberta_train_pool_cerf.png",
+      "complexity/outputs/plots/confusion_matrices/cm_roberta_train_pool_commonlit.png",
+      "complexity/outputs/plots/confusion_matrices/cm_roberta_train_pool_one_stop.png"
+    ],
+    "structured_data": [
+      "complexity/data/all-datasets-unified.parquet",
+      "complexity/data/asap-2-0.parquet",
+      "complexity/data/asap-aes.parquet",
+      "complexity/data/cambridge-english-readability.parquet",
+      "complexity/data/cerf-levelled-english-texts.parquet",
+      "complexity/data/commonlit-readability-prize.parquet",
+      "complexity/data/one-stop-corpus-english.parquet",
+      "complexity/data/train_pool_test.parquet",
+      "complexity/data/train_pool_train.parquet",
+      "complexity/data/train_pool_val.parquet",
+      "complexity/data/uk-key-stage-readability-for-english-texts.parquet"
+    ],
+    "tabular_data": [
+      "complexity/data/archive/asap-2-0/ASAP2_train_sourcetexts.csv",
+      "complexity/data/archive/cerf-levelled-english-texts/cefr_leveled_texts.csv",
+      "complexity/data/archive/uk-key-stage-readability-for-english-texts/TEST_balanced.csv",
+      "complexity/data/archive/uk-key-stage-readability-for-english-texts/TRAIN_balanced.csv",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/majority_class/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/tf_idf_logreg/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/tf_idf_svc/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/asap_aes/tf_idf_xgboost/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/majority_class/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/tf_idf_logreg/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/tf_idf_svc/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/cambridge/tf_idf_xgboost/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/majority_class/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/tf_idf_logreg/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/tf_idf_svc/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/cerf/tf_idf_xgboost/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/majority_class/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/tf_idf_logreg/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/tf_idf_svc/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/commonlit/tf_idf_xgboost/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/majority_class/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/tf_idf_logreg/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/tf_idf_svc/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/classical/one_stop/tf_idf_xgboost/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/asap_aes/glove_bilstm/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/asap_aes/glove_cnn/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/asap_aes/glove_mlp/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/cambridge/glove_bilstm/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/cambridge/glove_cnn/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/cambridge/glove_mlp/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/cerf/glove_bilstm/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/cerf/glove_cnn/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/cerf/glove_mlp/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/commonlit/glove_bilstm/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/commonlit/glove_cnn/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/commonlit/glove_mlp/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/one_stop/glove_bilstm/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/one_stop/glove_cnn/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/deep/one_stop/glove_mlp/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/family_summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/model_external_mean_latest.csv",
+      "complexity/outputs/cross_dataset_eval/plots/cross_dataset_matrix_f1_macro.csv",
+      "complexity/outputs/cross_dataset_eval/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/asap_aes/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/cambridge/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/cerf/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/commonlit/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/asap_aes/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/cambridge/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/cerf/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/commonlit/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/debertav3_train_pool/one_stop/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/asap_aes/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/cambridge/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/cerf/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/commonlit/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/distilbert_train_pool/one_stop/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/one_stop/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/asap_aes/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/cambridge/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/cerf/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/commonlit/summary_latest.csv",
+      "complexity/outputs/cross_dataset_eval/transformer/roberta_train_pool/one_stop/summary_latest.csv",
+      "complexity/outputs/metrics/baseline_curve_metrics.csv",
+      "complexity/outputs/metrics/baseline_eval_metrics.csv",
+      "complexity/outputs/metrics/classical_lc_metrics.csv",
+      "complexity/outputs/metrics/combined_curve_metrics.csv",
+      "complexity/outputs/metrics/deep_curve_metrics.csv",
+      "complexity/outputs/metrics/deep_eval_metrics_internal.csv",
+      "complexity/outputs/metrics/deep_eval_metrics_test.csv",
+      "complexity/outputs/metrics/deep_eval_metrics_val.csv",
+      "complexity/outputs/metrics/family_internal_summary.csv",
+      "complexity/outputs/metrics/final_transformer_error_analysis.csv",
+      "complexity/outputs/metrics/internal_family_summary_metrics.csv",
+      "complexity/outputs/metrics/internal_model_summary_metrics.csv",
+      "complexity/outputs/metrics/internal_test_leaderboard.csv",
+      "complexity/outputs/metrics/manual_example_predictions.csv",
+      "complexity/outputs/metrics/results_complexity-classification.csv",
+      "complexity/outputs/metrics/top10_test_accuracy.csv",
+      "complexity/outputs/metrics/top10_test_f1_macro.csv",
+      "complexity/outputs/metrics/top10_val_f1_macro.csv",
+      "complexity/outputs/metrics/transformer_summary_metrics.csv"
+    ]
+  },
+  "selected_dashboard_sources": [
+    {
+      "category": "metadata",
+      "path": "app/app-models/prod-model-complexity/model-config.yaml",
+      "reason": "Production model identity, runtime settings, and label mapping."
+    },
+    {
+      "category": "experiment_config",
+      "path": "complexity/outputs/metrics/cfg_20260310_2125.json",
+      "reason": "Experiment or evaluation config attached for traceability."
+    },
+    {
+      "category": "primary_evaluation",
+      "path": "complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_latest.json",
+      "reason": "Primary validation/test metrics for the production model."
+    },
+    {
+      "category": "benchmark",
+      "path": "complexity/outputs/metrics/results_complexity-classification.csv",
+      "reason": "Benchmark rows used to compare the production model against peers."
+    },
+    {
+      "category": "training_history",
+      "path": "complexity/logs/complexity-classification_GloVe_BiLSTM_[train_pool].log",
+      "reason": "Training-history artifact used to build learning and loss curves."
+    },
+    {
+      "category": "cross_dataset",
+      "path": "complexity/outputs/cross_dataset_eval/summary_latest.csv",
+      "reason": "Cross-dataset evaluation summary rows."
+    },
+    {
+      "category": "class_distribution",
+      "path": "complexity/data/train_pool_train.parquet",
+      "reason": "Explicit split artifacts used to derive class distributions."
+    },
+    {
+      "category": "class_distribution",
+      "path": "complexity/data/train_pool_val.parquet",
+      "reason": "Explicit split artifacts used to derive class distributions."
+    },
+    {
+      "category": "class_distribution",
+      "path": "complexity/data/train_pool_test.parquet",
+      "reason": "Explicit split artifacts used to derive class distributions."
+    },
+    {
+      "category": "prediction_samples",
+      "path": "complexity/outputs/metrics/manual_example_predictions.csv",
+      "reason": "Prediction samples used for qualitative dashboard review."
+    },
+    {
+      "category": "confusion_matrix",
+      "path": "complexity/outputs/plots/confusion_matrices/cm_glove_bilstm_train_pool.png",
+      "reason": "Confusion-matrix image copied into the dashboard bundle."
+    }
+  ],
+  "missing_dashboard_sources": [],
+  "notes": [
+    "Primary evaluation uses the production GloVe BiLSTM metrics_latest.json referenced by prod-model-complexity/model-config.yaml.",
+    "Complexity is the richest dashboard domain in the repo: internal benchmark data, external evaluation summaries, split parquets, manual sample predictions, and prod-model training logs are all present.",
+    "Confusion matrices exist only as saved images for the production model; no numeric matrix artifact was found.",
+    "class_distribution parsing failed for complexity/data/train_pool_train.parquet: Parquet dashboard sources require the optional pandas dependency.",
+    "class_distribution parsing failed for complexity/data/train_pool_val.parquet: Parquet dashboard sources require the optional pandas dependency.",
+    "class_distribution parsing failed for complexity/data/train_pool_test.parquet: Parquet dashboard sources require the optional pandas dependency."
+  ]
+}

label_classes.json

@@ -0,0 +1,5 @@
+[
+  "A1-A2",
+  "B1-B2",
+  "C1-C2"
+]

model-config.yaml

@@ -0,0 +1,103 @@
+model_id: complexity-bilstm-final-v1
+domain: complexity
+display_name: BiLSTM Final v1
+description: Final BiLSTM with additive attention for CEFR-based text complexity classification.
+is_active: true
+priority: 3
+framework:
+  type: pytorch
+  task: sequence-classification
+  library: torch
+  architecture: bilstm-attention
+  embeddings: fasttext-wiki-news-subwords-300
+artifacts:
+  weights:
+  - models/glove_bilstm_complexity_train_pool_20260307_163358/best_model/best.pt
+  tokenizer: []
+  config:
+  - outputs/metrics/cfg_20260310_2125.json
+  - models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_latest.json
+  vocabulary:
+  - models/shared/vocab.pkl
+  label_map_file: models/shared/class2id.json
+  label_classes_file: models/shared/label_classes.json
+  label_encoder_file: models/shared/label_encoder.pkl
+model:
+  embedding_dim: 300
+  hidden_dim: 128
+  num_layers: 2
+  dropout: 0.3
+  bidirectional: true
+  attention: additive
+  vocab_size: 50000
+  pad_token: <PAD>
+  unk_token: <UNK>
+runtime:
+  max_sequence_length: 256
+  truncation: true
+  padding: true
+  batch_size: 1
+  device: cpu
+  preprocessing: normalize_text + preprocess_from_normalized + texts_to_sequences
+dashboard:
+  builder: generic-v1
+  notes:
+  - Primary evaluation uses the production GloVe BiLSTM metrics_latest.json referenced
+    by prod-model-complexity/model-config.yaml.
+  - 'Complexity is the richest dashboard domain in the repo: internal benchmark data,
+    external evaluation summaries, split parquets, manual sample predictions, and
+    prod-model training logs are all present.'
+  - Confusion matrices exist only as saved images for the production model; no numeric
+    matrix artifact was found.
+  sources:
+    experiment_config: complexity/outputs/metrics/cfg_20260310_2125.json
+    primary_evaluation:
+      path: complexity/models/glove_bilstm_complexity_train_pool_20260307_163358/runs/metrics_latest.json
+      model_name: GloVe BiLSTM [train_pool]
+    benchmark:
+      path: complexity/outputs/metrics/results_complexity-classification.csv
+      model_name: GloVe BiLSTM [train_pool]
+    training_history: complexity/logs/complexity-classification_GloVe_BiLSTM_[train_pool].log
+    learning_curve:
+      status: not_applicable
+      reason: The complexity benchmark is built around a fixed train_pool dataset
+        rather than multiple prod-relevant train sizes.
+    cross_dataset:
+      path: complexity/outputs/cross_dataset_eval/summary_latest.csv
+      model_name: GloVe BiLSTM
+    class_distribution:
+      train: complexity/data/train_pool_train.parquet
+      val: complexity/data/train_pool_val.parquet
+      test: complexity/data/train_pool_test.parquet
+      label_field: label
+      dataset_field: source_dataset
+    prediction_samples:
+      path: complexity/outputs/metrics/manual_example_predictions.csv
+      production_prediction:
+        model: GloVe BiLSTM
+        label_field: GloVe_BiLSTM_label
+        confidence_field: GloVe_BiLSTM_conf
+      reference_prediction:
+        model: DeBERTaV3
+        label_field: DeBERTaV3_label
+        confidence_field: DeBERTaV3_conf
+    confusion_matrix: complexity/outputs/plots/confusion_matrices/cm_glove_bilstm_train_pool.png
+inference:
+  return_probabilities: true
+  confidence_method: softmax_max
+ui:
+  domain_display_name: Text Complexity
+  color_token: complexity
+  group: complexity-core
+labels:
+  type: single-label-classification
+  classes:
+  - id: 0
+    name: A1-A2
+    display_name: A1-A2
+  - id: 1
+    name: B1-B2
+    display_name: B1-B2
+  - id: 2
+    name: C1-C2
+    display_name: C1-C2

test_inference.ipynb

@@ -0,0 +1,456 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "af313b4f",
+   "metadata": {},
+   "source": [
+    "# Complexity Model Inference Test\n",
+    "\n",
+    "This notebook loads the saved production complexity model and its local artifacts from this folder, rebuilds the BiLSTM architecture, and runs inference on a few example sentences. It performs inference only."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9976ef3c",
+   "metadata": {},
+   "source": [
+    "## 1. Load the local artifacts\n",
+    "\n",
+    "The code below loads the checkpoint, model config, vocabulary, and label classes directly from the current folder."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "635fe6e7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model directory: /Users/georgijkutivadze/projects/mago-text-scoring/app/app-models/prod-model-complexity\n",
+      "Device: cpu\n",
+      "Artifacts: ['best.pt', 'label_classes.json', 'model-config.yaml', 'test_inference.ipynb', 'vocab.pkl']\n",
+      "Checkpoint keys: ['epoch', 'model_state', 'optimizer_state', 'scheduler_state', 'best_f1', 'history', 'model_name', 'task']\n",
+      "Embedding matrix shape: (50000, 300)\n",
+      "Vocabulary size: 50000\n",
+      "Labels: ['A1-A2', 'B1-B2', 'C1-C2']\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "import json\n",
+    "import pickle\n",
+    "import re\n",
+    "import unicodedata\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import yaml\n",
+    "\n",
+    "MODEL_DIR = Path(\".\").resolve()\n",
+    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "\n",
+    "with (MODEL_DIR / \"model-config.yaml\").open(\"r\", encoding=\"utf-8\") as handle:\n",
+    "    model_config = yaml.safe_load(handle)\n",
+    "\n",
+    "with (MODEL_DIR / \"label_classes.json\").open(\"r\", encoding=\"utf-8\") as handle:\n",
+    "    label_classes = json.load(handle)\n",
+    "\n",
+    "with (MODEL_DIR / \"vocab.pkl\").open(\"rb\") as handle:\n",
+    "    vocab = pickle.load(handle)\n",
+    "\n",
+    "checkpoint = torch.load(MODEL_DIR / \"best.pt\", map_location=device)\n",
+    "state_dict = checkpoint[\"model_state\"] if isinstance(checkpoint, dict) and \"model_state\" in checkpoint else checkpoint\n",
+    "embedding_matrix = state_dict[\"embedding.weight\"].detach().cpu().numpy()\n",
+    "artifact_files = sorted(path.name for path in MODEL_DIR.iterdir() if path.is_file())\n",
+    "\n",
+    "print(f\"Model directory: {MODEL_DIR}\")\n",
+    "print(f\"Device: {device}\")\n",
+    "print(f\"Artifacts: {artifact_files}\")\n",
+    "print(f\"Checkpoint keys: {list(checkpoint.keys()) if isinstance(checkpoint, dict) else 'state_dict'}\")\n",
+    "print(f\"Embedding matrix shape: {embedding_matrix.shape}\")\n",
+    "print(f\"Vocabulary size: {len(vocab)}\")\n",
+    "print(f\"Labels: {label_classes}\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "abe5a2e3",
+   "metadata": {},
+   "source": [
+    "## 2. Recreate the model and preprocessing pipeline\n",
+    "\n",
+    "The notebook rebuilds the saved BiLSTM + additive-attention classifier and applies the lightweight text normalization used for inference."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "f86fa8a2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class AdditiveAttention(nn.Module):\n",
+    "    def __init__(self, hidden_dim: int):\n",
+    "        super().__init__()\n",
+    "        self.attn = nn.Linear(hidden_dim, 1, bias=False)\n",
+    "\n",
+    "    def forward(self, hidden_states, mask=None):\n",
+    "        scores = self.attn(hidden_states).squeeze(-1)\n",
+    "        if mask is not None:\n",
+    "            scores = scores.masked_fill(~mask, -1e9)\n",
+    "        weights = torch.softmax(scores, dim=1)\n",
+    "        context = (weights.unsqueeze(-1) * hidden_states).sum(dim=1)\n",
+    "        return context, weights\n",
+    "\n",
+    "\n",
+    "class BiLSTMAttention(nn.Module):\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        embedding_matrix: np.ndarray,\n",
+    "        num_classes: int,\n",
+    "        hidden_dim: int = 128,\n",
+    "        num_layers: int = 2,\n",
+    "        dropout: float = 0.3,\n",
+    "    ):\n",
+    "        super().__init__()\n",
+    "        _, embedding_dim = embedding_matrix.shape\n",
+    "        self.embedding = nn.Embedding.from_pretrained(\n",
+    "            torch.tensor(embedding_matrix, dtype=torch.float32),\n",
+    "            freeze=False,\n",
+    "            padding_idx=0,\n",
+    "        )\n",
+    "        self.lstm = nn.LSTM(\n",
+    "            input_size=embedding_dim,\n",
+    "            hidden_size=hidden_dim,\n",
+    "            num_layers=num_layers,\n",
+    "            batch_first=True,\n",
+    "            bidirectional=True,\n",
+    "            dropout=dropout if num_layers > 1 else 0.0,\n",
+    "        )\n",
+    "        self.attention = AdditiveAttention(hidden_dim * 2)\n",
+    "        self.dropout = nn.Dropout(dropout)\n",
+    "        self.fc = nn.Linear(hidden_dim * 2, num_classes)\n",
+    "\n",
+    "    def forward(self, token_ids):\n",
+    "        mask = token_ids != 0\n",
+    "        embeddings = self.dropout(self.embedding(token_ids))\n",
+    "        hidden_states, _ = self.lstm(embeddings)\n",
+    "        context, _ = self.attention(hidden_states, mask)\n",
+    "        return self.fc(self.dropout(context))\n",
+    "\n",
+    "\n",
+    "DEFAULT_STOPWORDS = {\n",
+    "    \"a\", \"an\", \"and\", \"are\", \"as\", \"at\", \"be\", \"by\", \"for\", \"from\",\n",
+    "    \"has\", \"he\", \"in\", \"is\", \"it\", \"its\", \"of\", \"on\", \"that\", \"the\",\n",
+    "    \"to\", \"was\", \"were\", \"will\", \"with\"\n",
+    "}\n",
+    "\n",
+    "try:\n",
+    "    from nltk.corpus import stopwords\n",
+    "    STOPWORDS = set(stopwords.words(\"english\"))\n",
+    "except Exception:\n",
+    "    STOPWORDS = DEFAULT_STOPWORDS\n",
+    "\n",
+    "try:\n",
+    "    from nltk.stem import WordNetLemmatizer\n",
+    "    LEMMATIZER = WordNetLemmatizer()\n",
+    "except Exception:\n",
+    "    LEMMATIZER = None\n",
+    "\n",
+    "\n",
+    "def maybe_lemmatize(token: str) -> str:\n",
+    "    if LEMMATIZER is None:\n",
+    "        return token\n",
+    "    try:\n",
+    "        return LEMMATIZER.lemmatize(token)\n",
+    "    except Exception:\n",
+    "        return token\n",
+    "\n",
+    "\n",
+    "def normalize_text(text: str) -> str:\n",
+    "    text = str(text).lower()\n",
+    "    text = unicodedata.normalize(\"NFKD\", text)\n",
+    "    text = re.sub(r\"http\\S+|www\\S+\", \" \", text)\n",
+    "    text = re.sub(r\"@\\w+|#\\w+\", \" \", text)\n",
+    "    text = re.sub(r\"[^a-zA-Z0-9\\s!?.,'\\\"]\", \" \", text)\n",
+    "    text = re.sub(r\"\\s+\", \" \", text).strip()\n",
+    "    return text\n",
+    "\n",
+    "\n",
+    "def preprocess_from_normalized(text: str) -> str:\n",
+    "    tokens = [token for token in text.split() if token not in STOPWORDS]\n",
+    "    return \" \".join(maybe_lemmatize(token) for token in tokens)\n",
+    "\n",
+    "\n",
+    "def texts_to_sequences(texts, vocab_map, max_len, unk_token=\"<UNK>\"):\n",
+    "    unk_idx = vocab_map.get(unk_token, 1)\n",
+    "    sequences = np.zeros((len(texts), max_len), dtype=np.int64)\n",
+    "    for row_idx, text in enumerate(texts):\n",
+    "        for col_idx, token in enumerate(str(text).split()[:max_len]):\n",
+    "            sequences[row_idx, col_idx] = vocab_map.get(token, unk_idx)\n",
+    "    return sequences\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0c36b0e4",
+   "metadata": {},
+   "source": [
+    "## 3. Restore the checkpoint and run example inference\n",
+    "\n",
+    "This cell rebuilds the model, restores the saved weights, preprocesses example sentences, and displays the predicted complexity label with probabilities."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "229ce4ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "runtime_config = model_config.get(\"runtime\", {})\n",
+    "network_config = model_config.get(\"model\", {})\n",
+    "\n",
+    "model = BiLSTMAttention(\n",
+    "    embedding_matrix=embedding_matrix,\n",
+    "    num_classes=len(label_classes),\n",
+    "    hidden_dim=network_config.get(\"hidden_dim\", 128),\n",
+    "    num_layers=network_config.get(\"num_layers\", 2),\n",
+    "    dropout=network_config.get(\"dropout\", 0.3),\n",
+    ")\n",
+    "model.load_state_dict(state_dict)\n",
+    "model.to(device)\n",
+    "model.eval()\n",
+    "\n",
+    "MAX_LEN = runtime_config.get(\"max_sequence_length\", 256)\n",
+    "\n",
+    "\n",
+    "@torch.inference_mode()\n",
+    "def predict_texts(texts):\n",
+    "    normalized = [normalize_text(text) for text in texts]\n",
+    "    preprocessed = [preprocess_from_normalized(text) for text in normalized]\n",
+    "    sequences = texts_to_sequences(preprocessed, vocab, MAX_LEN)\n",
+    "    inputs = torch.tensor(sequences, dtype=torch.long, device=device)\n",
+    "    logits = model(inputs)\n",
+    "    probabilities = torch.softmax(logits, dim=-1).cpu().numpy()\n",
+    "\n",
+    "    rows = []\n",
+    "    for raw_text, normalized_text, preprocessed_text, probs in zip(texts, normalized, preprocessed, probabilities):\n",
+    "        pred_idx = int(probs.argmax())\n",
+    "        rows.append(\n",
+    "            {\n",
+    "                \"text\": raw_text,\n",
+    "                \"normalized_text\": normalized_text,\n",
+    "                \"preprocessed_text\": preprocessed_text,\n",
+    "                \"predicted_label\": label_classes[pred_idx],\n",
+    "                \"confidence\": round(float(probs[pred_idx]), 4),\n",
+    "                \"probabilities\": {\n",
+    "                    label: round(float(score), 4)\n",
+    "                    for label, score in zip(label_classes, probs)\n",
+    "                },\n",
+    "            }\n",
+    "        )\n",
+    "\n",
+    "    return pd.DataFrame(rows)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "62c8eeb8-3663-40f7-9b75-20ba0e6017db",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>text</th>\n",
+       "      <th>normalized_text</th>\n",
+       "      <th>preprocessed_text</th>\n",
+       "      <th>predicted_label</th>\n",
+       "      <th>confidence</th>\n",
+       "      <th>probabilities</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>I have a dog. It is small and very cute. We play in the park every day.</td>\n",
+       "      <td>i have a dog. it is small and very cute. we play in the park every day.</td>\n",
+       "      <td>dog. small cute. play park every day.</td>\n",
+       "      <td>A1-A2</td>\n",
+       "      <td>0.6027</td>\n",
+       "      <td>{'A1-A2': 0.6027, 'B1-B2': 0.3619, 'C1-C2': 0.0354}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>My name is Anna. I am a student. I live with my family in a big city.</td>\n",
+       "      <td>my name is anna. i am a student. i live with my family in a big city.</td>\n",
+       "      <td>name anna. student. live family big city.</td>\n",
+       "      <td>C1-C2</td>\n",
+       "      <td>0.7338</td>\n",
+       "      <td>{'A1-A2': 0.1452, 'B1-B2': 0.121, 'C1-C2': 0.7338}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Many people enjoy traveling by train because it is comfortable, affordable, and less stressful than flying.</td>\n",
+       "      <td>many people enjoy traveling by train because it is comfortable, affordable, and less stressful than flying.</td>\n",
+       "      <td>many people enjoy traveling train comfortable, affordable, less stressful flying.</td>\n",
+       "      <td>C1-C2</td>\n",
+       "      <td>0.8663</td>\n",
+       "      <td>{'A1-A2': 0.0018, 'B1-B2': 0.1319, 'C1-C2': 0.8663}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>If students manage their time well, they can improve their grades and still have enough time for rest and hobbies.</td>\n",
+       "      <td>if students manage their time well, they can improve their grades and still have enough time for rest and hobbies.</td>\n",
+       "      <td>student manage time well, improve grade still enough time rest hobbies.</td>\n",
+       "      <td>B1-B2</td>\n",
+       "      <td>0.9152</td>\n",
+       "      <td>{'A1-A2': 0.0168, 'B1-B2': 0.9152, 'C1-C2': 0.068}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>The committee’s recommendation, while ostensibly pragmatic, ultimately fails to address the structural asymmetries t...</td>\n",
+       "      <td>the committee s recommendation, while ostensibly pragmatic, ultimately fails to address the structural asymmetries t...</td>\n",
+       "      <td>committee recommendation, ostensibly pragmatic, ultimately fails address structural asymmetry perpetuate institution...</td>\n",
+       "      <td>C1-C2</td>\n",
+       "      <td>0.9565</td>\n",
+       "      <td>{'A1-A2': 0.0023, 'B1-B2': 0.0412, 'C1-C2': 0.9565}</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>Her argument is particularly compelling because it synthesizes historical evidence, theoretical precision, and a sop...</td>\n",
+       "      <td>her argument is particularly compelling because it synthesizes historical evidence, theoretical precision, and a sop...</td>\n",
+       "      <td>argument particularly compelling synthesizes historical evidence, theoretical precision, sophisticated awareness cul...</td>\n",
+       "      <td>B1-B2</td>\n",
+       "      <td>0.6361</td>\n",
+       "      <td>{'A1-A2': 0.2443, 'B1-B2': 0.6361, 'C1-C2': 0.1197}</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                      text  \\\n",
+       "0                                                  I have a dog. It is small and very cute. We play in the park every day.   \n",
+       "1                                                    My name is Anna. I am a student. I live with my family in a big city.   \n",
+       "2              Many people enjoy traveling by train because it is comfortable, affordable, and less stressful than flying.   \n",
+       "3       If students manage their time well, they can improve their grades and still have enough time for rest and hobbies.   \n",
+       "4  The committee’s recommendation, while ostensibly pragmatic, ultimately fails to address the structural asymmetries t...   \n",
+       "5  Her argument is particularly compelling because it synthesizes historical evidence, theoretical precision, and a sop...   \n",
+       "\n",
+       "                                                                                                           normalized_text  \\\n",
+       "0                                                  i have a dog. it is small and very cute. we play in the park every day.   \n",
+       "1                                                    my name is anna. i am a student. i live with my family in a big city.   \n",
+       "2              many people enjoy traveling by train because it is comfortable, affordable, and less stressful than flying.   \n",
+       "3       if students manage their time well, they can improve their grades and still have enough time for rest and hobbies.   \n",
+       "4  the committee s recommendation, while ostensibly pragmatic, ultimately fails to address the structural asymmetries t...   \n",
+       "5  her argument is particularly compelling because it synthesizes historical evidence, theoretical precision, and a sop...   \n",
+       "\n",
+       "                                                                                                         preprocessed_text  \\\n",
+       "0                                                                                    dog. small cute. play park every day.   \n",
+       "1                                                                                name anna. student. live family big city.   \n",
+       "2                                        many people enjoy traveling train comfortable, affordable, less stressful flying.   \n",
+       "3                                                  student manage time well, improve grade still enough time rest hobbies.   \n",
+       "4  committee recommendation, ostensibly pragmatic, ultimately fails address structural asymmetry perpetuate institution...   \n",
+       "5  argument particularly compelling synthesizes historical evidence, theoretical precision, sophisticated awareness cul...   \n",
+       "\n",
+       "  predicted_label  confidence  \\\n",
+       "0           A1-A2      0.6027   \n",
+       "1           C1-C2      0.7338   \n",
+       "2           C1-C2      0.8663   \n",
+       "3           B1-B2      0.9152   \n",
+       "4           C1-C2      0.9565   \n",
+       "5           B1-B2      0.6361   \n",
+       "\n",
+       "                                         probabilities  \n",
+       "0  {'A1-A2': 0.6027, 'B1-B2': 0.3619, 'C1-C2': 0.0354}  \n",
+       "1   {'A1-A2': 0.1452, 'B1-B2': 0.121, 'C1-C2': 0.7338}  \n",
+       "2  {'A1-A2': 0.0018, 'B1-B2': 0.1319, 'C1-C2': 0.8663}  \n",
+       "3   {'A1-A2': 0.0168, 'B1-B2': 0.9152, 'C1-C2': 0.068}  \n",
+       "4  {'A1-A2': 0.0023, 'B1-B2': 0.0412, 'C1-C2': 0.9565}  \n",
+       "5  {'A1-A2': 0.2443, 'B1-B2': 0.6361, 'C1-C2': 0.1197}  "
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pd.set_option(\"display.max_colwidth\", 120)\n",
+    "\n",
+    "example_sentences = [\n",
+    "    # A1-A2\n",
+    "    \"I have a dog. It is small and very cute. We play in the park every day.\",\n",
+    "    \"My name is Anna. I am a student. I live with my family in a big city.\",\n",
+    "\n",
+    "    # B1-B2\n",
+    "    \"Many people enjoy traveling by train because it is comfortable, affordable, and less stressful than flying.\",\n",
+    "    \"If students manage their time well, they can improve their grades and still have enough time for rest and hobbies.\",\n",
+    "\n",
+    "    # C1-C2\n",
+    "    \"The committee’s recommendation, while ostensibly pragmatic, ultimately fails to address the structural asymmetries that perpetuate institutional inefficiency.\",\n",
+    "    \"Her argument is particularly compelling because it synthesizes historical evidence, theoretical precision, and a sophisticated awareness of cultural nuance.\"\n",
+    "]\n",
+    "\n",
+    "results = predict_texts(example_sentences)\n",
+    "results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "31f6fc98-7438-4e16-a8aa-87e63c61ee6e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

vocab.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbce8dfb87bf30450435d0e38f437b9b34c97b3a7d6faa485ce3e605c5a90278
+size 675803