{ "cells": [ { "cell_type": "code", "execution_count": 14, "id": "71fa50b4-5593-4a9b-9c2f-7da7c76e82b7", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\Admin\\.cache\\huggingface\\modules\\transformers_modules\\zhihan1996\\DNABERT-2-117M\\7bce263b15377fc15361f52cfab88f8b586abda0\\bert_layers.py:126: UserWarning: Unable to import Triton; defaulting MosaicBERT attention implementation to pytorch (this will reduce throughput when using this model).\n", " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Saved annotated_result.csv\n" ] } ], "source": [ "import pandas as pd\n", "import re\n", "import torch\n", "from transformers import BertConfig, AutoTokenizer, AutoModelForSequenceClassification\n", "\n", "# 1) Load and clean clustering results\n", "df = pd.read_csv(\"result.csv\")\n", "df['Fragment'] = df['Fragment'].astype(str).apply(lambda x: re.sub(r'[^ACGT]', '', x))\n", "df = df[(df['Fragment'] != '') & df['Fragment'].notna()]\n", "\n", "# Optional: keep original clustering labels (1,2,3)\n", "if 'label' in df.columns:\n", " df['cluster_label_1idx'] = df['label']\n", " df['cluster_label_0idx'] = df['label'] - 1\n", "\n", "# 2) Load model and tokenizer consistently from the same repo\n", "MODEL_NAME = \"Priyasi/MetaTrans_CLM\"\n", "config = BertConfig.from_pretrained(MODEL_NAME, num_labels=3)\n", "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)\n", "model = AutoModelForSequenceClassification.from_pretrained(\n", " MODEL_NAME,\n", " config=config,\n", " trust_remote_code=True,\n", " torch_dtype=\"auto\",\n", ")\n", "model.eval()\n", "\n", "# 3) Predict annotated labels\n", "enc = tokenizer(df['Fragment'].tolist(), truncation=True, padding=True, max_length=512, return_tensors=\"pt\")\n", "with torch.no_grad():\n", " logits = model(**enc).logits\n", " preds = torch.argmax(logits, dim=-1).cpu().numpy()\n", "\n", "df['annotated_label'] = preds # 0-indexed (0,1,2)\n", "df.to_csv(\"annotated_result.csv\", index=False)\n", "print(\"Saved annotated_result.csv\")\n" ] }, { "cell_type": "code", "execution_count": 25, "id": "a11bb5ea-212e-4bf7-9f06-83d51f1bd7f0", "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", "from transformers import TrainingArguments, Trainer\n", "import numpy as np\n", "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n", "import torch\n", "\n", "# Use the model’s own num_labels unless you truly need to change it\n", "num_labels = getattr(config, \"num_labels\", None)\n", "if num_labels is None:\n", " num_labels = len(set(df['annotated_label']))\n", " config.num_labels = num_labels # keep config and model aligned\n", "\n", "# Replace labels with annotations\n", "df_train = df.dropna(subset=['Fragment', 'annotated_label']).copy()\n", "df_train['label'] = df_train['annotated_label']\n", "\n", "# Split\n", "train_df, val_df = train_test_split(\n", " df_train, test_size=0.2, random_state=42, stratify=df_train['label']\n", ")\n", "\n", "train_texts = train_df['Fragment'].tolist()\n", "train_labels = train_df['label'].tolist()\n", "val_texts = val_df['Fragment'].tolist()\n", "val_labels = val_df['label'].tolist()\n", "\n", "# Tokenize\n", "train_enc = tokenizer(train_texts, truncation=True, padding=True, max_length=512)\n", "val_enc = tokenizer(val_texts, truncation=True, padding=True, max_length=512)\n", "\n", "class MetaData(torch.utils.data.Dataset):\n", " def __init__(self, encodings, labels):\n", " self.encodings = encodings\n", " self.labels = labels\n", " def __getitem__(self, idx):\n", " item = {k: torch.tensor(v[idx]) for k, v in self.encodings.items()}\n", " item['labels'] = torch.tensor(self.labels[idx])\n", " return item\n", " def __len__(self):\n", " return len(self.labels)\n", "\n", "train_ds = MetaData(train_enc, train_labels)\n", "val_ds = MetaData(val_enc, val_labels)\n", "\n", "# Metrics\n", "def compute_metrics(pred):\n", " labels = pred.label_ids\n", " logits = pred.predictions[0] if isinstance(pred.predictions, tuple) else pred.predictions\n", " preds = np.argmax(logits, axis=-1)\n", " return {\n", " \"accuracy\": accuracy_score(labels, preds),\n", " \"precision\": precision_score(labels, preds, average=\"weighted\"),\n", " \"recall\": recall_score(labels, preds, average=\"weighted\"),\n", " \"f1\": f1_score(labels, preds, average=\"weighted\"),\n", " }\n" ] }, { "cell_type": "code", "execution_count": 31, "id": "7b7225e9-dce1-498b-8207-ce203c6c83c9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\Admin\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\transformers\\training_args.py:1568: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n" ] } ], "source": [ "\n", "training_args = TrainingArguments(\n", " output_dir='./results_clm_star',\n", " num_train_epochs=10, \n", " per_device_train_batch_size=16,\n", " per_device_eval_batch_size=32,\n", " warmup_steps=10,\n", " weight_decay=0.01,\n", " learning_rate=2e-5,\n", " logging_dir='./logs_clm_star',\n", " logging_steps=50,\n", " evaluation_strategy=\"epoch\",\n", " save_strategy=\"epoch\",\n", " load_best_model_at_end=True,\n", " metric_for_best_model=\"f1\",\n", " report_to=\"none\",\n", ")\n", "\n", "\n", "trainer = Trainer(\n", " model=model,\n", " args=training_args, # ✅ use the TrainingArguments you defined\n", " train_dataset=train_ds,\n", " eval_dataset=val_ds,\n", " compute_metrics=compute_metrics,\n", ")\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 32, "id": "68f04416-7d98-4e23-b013-d9e3f769084b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- Starting CLM* fine-tuning on annotated dataset ---\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [750/750 03:51, Epoch 10/10]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossAccuracyPrecisionRecallF1
10.0296000.5414890.9266670.9313560.9266670.926021
20.0229000.6936670.9166670.9223300.9166670.915891
30.0252000.5331310.9300000.9312100.9300000.929878
40.0758000.3104110.9533330.9544600.9533330.953119
50.0579000.3242360.9400000.9416700.9400000.939695
60.0248000.3739280.9433330.9465710.9433330.943028
70.0159000.2606950.9633330.9638540.9633330.963220
80.0008000.4253100.9433330.9451080.9433330.943054
90.0002000.3361960.9533330.9543050.9533330.953182
100.0014000.3411370.9500000.9508680.9500000.949849

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "TrainOutput(global_step=750, training_loss=0.025374397090325754, metrics={'train_runtime': 232.1361, 'train_samples_per_second': 51.694, 'train_steps_per_second': 3.231, 'total_flos': 328099930560000.0, 'train_loss': 0.025374397090325754, 'epoch': 10.0})" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "print(\"--- Starting CLM* fine-tuning on annotated dataset ---\")\n", "trainer.train()\n", "\n" ] }, { "cell_type": "code", "execution_count": 33, "id": "3fb4cd51-67e9-4cc2-901b-33fce8b95585", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Saved CLM* to ./Model_CLM_star\n" ] } ], "source": [ "# Save CLM*\n", "save_dir = \"./Model_CLM_star\"\n", "model.save_pretrained(save_dir)\n", "tokenizer.save_pretrained(save_dir)\n", "print(f\"Saved CLM* to {save_dir}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "7cda7294-57e9-4587-8144-911ca0fc91e5", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python (torch-gpu)", "language": "python", "name": "torch-gpu" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }