add programs

TRAIN_GPT.ipynb

@@ -0,0 +1,1214 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "view-in-github"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/gist/LencoDigitexer/b1f3602547b99e3bc7478c3188c65024/.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0RFUxrlMNIkx"
+      },
+      "source": [
+        "#Импорты"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "metadata": {
+        "id": "0lhkuNdOoqvq"
+      },
+      "outputs": [],
+      "source": [
+        "import pandas as pd\n",
+        "import numpy as np"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "d8fP5y2Ipsln",
+        "outputId": "1096c5a5-7ef6-401c-b0d9-b59e6cf9fc86"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install transformers==4.19.0"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 24,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Requirement already satisfied: transformers in c:\\github\\train_gpt\\.conda\\lib\\site-packages (4.19.0)\n",
+            "Collecting transformers\n",
+            "  Downloading transformers-4.36.1-py3-none-any.whl.metadata (126 kB)\n",
+            "     ---------------------------------------- 0.0/126.8 kB ? eta -:--:--\n",
+            "     --- ------------------------------------ 10.2/126.8 kB ? eta -:--:--\n",
+            "     --- ------------------------------------ 10.2/126.8 kB ? eta -:--:--\n",
+            "     -------- ---------------------------- 30.7/126.8 kB 145.2 kB/s eta 0:00:01\n",
+            "     -------- ---------------------------- 30.7/126.8 kB 145.2 kB/s eta 0:00:01\n",
+            "     ----------------- ------------------- 61.4/126.8 kB 233.8 kB/s eta 0:00:01\n",
+            "     ------------------------------- ---- 112.6/126.8 kB 364.4 kB/s eta 0:00:01\n",
+            "     ------------------------------------ 126.8/126.8 kB 373.4 kB/s eta 0:00:00\n",
+            "Requirement already satisfied: filelock in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from transformers) (3.13.1)\n",
+            "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from transformers) (0.19.4)\n",
+            "Requirement already satisfied: numpy>=1.17 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from transformers) (1.26.1)\n",
+            "Requirement already satisfied: packaging>=20.0 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from transformers) (23.0)\n",
+            "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from transformers) (6.0.1)\n",
+            "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from transformers) (2023.10.3)\n",
+            "Requirement already satisfied: requests in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from transformers) (2.28.2)\n",
+            "Collecting tokenizers<0.19,>=0.14 (from transformers)\n",
+            "  Using cached tokenizers-0.15.0-cp311-none-win_amd64.whl.metadata (6.8 kB)\n",
+            "Requirement already satisfied: safetensors>=0.3.1 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from transformers) (0.4.0)\n",
+            "Requirement already satisfied: tqdm>=4.27 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from transformers) (4.66.1)\n",
+            "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (2023.10.0)\n",
+            "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.8.0)\n",
+            "Requirement already satisfied: colorama in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from tqdm>=4.27->transformers) (0.4.6)\n",
+            "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from requests->transformers) (3.1.0)\n",
+            "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from requests->transformers) (3.4)\n",
+            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from requests->transformers) (1.26.15)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\addmi\\appdata\\roaming\\python\\python311\\site-packages (from requests->transformers) (2022.12.7)\n",
+            "Downloading transformers-4.36.1-py3-none-any.whl (8.3 MB)\n",
+            "   ---------------------------------------- 0.0/8.3 MB ? eta -:--:--\n",
+            "    --------------------------------------- 0.1/8.3 MB 3.3 MB/s eta 0:00:03\n",
+            "    --------------------------------------- 0.2/8.3 MB 2.0 MB/s eta 0:00:05\n",
+            "   - -------------------------------------- 0.2/8.3 MB 2.2 MB/s eta 0:00:04\n",
+            "   - -------------------------------------- 0.3/8.3 MB 1.8 MB/s eta 0:00:05\n",
+            "   -- ------------------------------------- 0.5/8.3 MB 2.5 MB/s eta 0:00:04\n",
+            "   ---- ----------------------------------- 0.8/8.3 MB 3.0 MB/s eta 0:00:03\n",
+            "   ----- ---------------------------------- 1.2/8.3 MB 3.7 MB/s eta 0:00:02\n",
+            "   -------- ------------------------------- 1.7/8.3 MB 4.6 MB/s eta 0:00:02\n",
+            "   ---------- ----------------------------- 2.2/8.3 MB 5.3 MB/s eta 0:00:02\n",
+            "   ------------ --------------------------- 2.6/8.3 MB 5.5 MB/s eta 0:00:02\n",
+            "   --------------- ------------------------ 3.2/8.3 MB 5.9 MB/s eta 0:00:01\n",
+            "   --------------- ------------------------ 3.2/8.3 MB 6.0 MB/s eta 0:00:01\n",
+            "   --------------- ------------------------ 3.2/8.3 MB 6.0 MB/s eta 0:00:01\n",
+            "   --------------- ------------------------ 3.2/8.3 MB 6.0 MB/s eta 0:00:01\n",
+            "   --------------- ------------------------ 3.2/8.3 MB 6.0 MB/s eta 0:00:01\n",
+            "   ---------------- ----------------------- 3.5/8.3 MB 4.7 MB/s eta 0:00:02\n",
+            "   ---------------- ----------------------- 3.5/8.3 MB 4.7 MB/s eta 0:00:02\n",
+            "   ------------------ --------------------- 3.8/8.3 MB 4.4 MB/s eta 0:00:02\n",
+            "   --------------------- ------------------ 4.5/8.3 MB 5.2 MB/s eta 0:00:01\n",
+            "   ---------------------------- ----------- 5.9/8.3 MB 6.2 MB/s eta 0:00:01\n",
+            "   ------------------------------- -------- 6.5/8.3 MB 6.4 MB/s eta 0:00:01\n",
+            "   --------------------------------- ------ 6.9/8.3 MB 6.5 MB/s eta 0:00:01\n",
+            "   ----------------------------------- ---- 7.4/8.3 MB 6.7 MB/s eta 0:00:01\n",
+            "   ------------------------------------- -- 7.8/8.3 MB 6.8 MB/s eta 0:00:01\n",
+            "   ---------------------------------------  8.3/8.3 MB 7.0 MB/s eta 0:00:01\n",
+            "   ---------------------------------------- 8.3/8.3 MB 6.8 MB/s eta 0:00:00\n",
+            "Using cached tokenizers-0.15.0-cp311-none-win_amd64.whl (2.2 MB)\n",
+            "Installing collected packages: tokenizers, transformers\n",
+            "  Attempting uninstall: tokenizers\n",
+            "    Found existing installation: tokenizers 0.12.1\n",
+            "    Uninstalling tokenizers-0.12.1:\n",
+            "      Successfully uninstalled tokenizers-0.12.1\n",
+            "  Attempting uninstall: transformers\n",
+            "    Found existing installation: transformers 4.19.0\n",
+            "    Uninstalling transformers-4.19.0:\n",
+            "      Successfully uninstalled transformers-4.19.0\n",
+            "Successfully installed tokenizers-0.15.0 transformers-4.36.1\n"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "  WARNING: Failed to remove contents in a temporary directory 'C:\\GitHub\\TRAIN_GPT\\.conda\\Lib\\site-packages\\~okenizers'.\n",
+            "  You can safely remove it manually.\n"
+          ]
+        }
+      ],
+      "source": [
+        "!pip install -U transformers"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 25,
+      "metadata": {
+        "id": "O6ujFUYBl2Fj"
+      },
+      "outputs": [],
+      "source": [
+        "from transformers import GPT2LMHeadModel, GPT2Tokenizer\n",
+        "import torch\n",
+        "from torch.utils.data import Dataset # this is the pytorch class import\n",
+        "from transformers import DataCollatorForLanguageModeling\n",
+        "from transformers import TrainingArguments, Trainer\n",
+        "from transformers import StoppingCriteria, StoppingCriteriaList"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 26,
+      "metadata": {
+        "id": "k7cC6BjGmhWm"
+      },
+      "outputs": [],
+      "source": [
+        "DEVICE = torch.device(\"cuda\") if torch.cuda.is_available() else None"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "BFdrbagdcFSQ",
+        "outputId": "8d98ad2f-dd04-4d5a-ab15-1cfbfcb06f79"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install openpyxl"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4Br1yvNgNBCx"
+      },
+      "source": [
+        "\n",
+        "#Обучение модельки"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 27,
+      "metadata": {
+        "id": "3ZtL62M80QJF"
+      },
+      "outputs": [],
+      "source": [
+        "my_path = 'content/'"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "PDtJ6TVDpyVr",
+        "outputId": "1fa22c1d-64f9-4a5d-d046-8f64a57a2bc9"
+      },
+      "outputs": [],
+      "source": [
+        "!wget https://www.dropbox.com/s/89rmm4wucjve2ll/news.csv?dl=0 -O /content/news.csv"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 28,
+      "metadata": {
+        "id": "_oaLt4broJKd"
+      },
+      "outputs": [],
+      "source": [
+        "data = pd.read_csv(f'{my_path}news.csv')\n",
+        "data.drop_duplicates(subset = ['title'],inplace=True)\n",
+        "data.dropna(inplace=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 29,
+      "metadata": {
+        "id": "X6JZphFn_WlM"
+      },
+      "outputs": [],
+      "source": [
+        "def clear_data(text: str):\n",
+        "  if 'Что такое' in text:\n",
+        "    return np.nan\n",
+        "  if len( text.split(' ')) < 20:\n",
+        "    return np.nan\n",
+        "  return text"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 30,
+      "metadata": {
+        "id": "DWOQE6P2AdW2"
+      },
+      "outputs": [],
+      "source": [
+        "data['text'] = data['text'].apply(clear_data)\n",
+        "data.dropna(subset=['text'], inplace=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "C9P12BUCgshN",
+        "outputId": "fbb04d40-6f50-4b65-e26d-2d033fbc745c"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Fri Jan 20 17:48:31 2023       \n",
+            "+-----------------------------------------------------------------------------+\n",
+            "| NVIDIA-SMI 516.94       Driver Version: 516.94       CUDA Version: 11.7     |\n",
+            "|-------------------------------+----------------------+----------------------+\n",
+            "| GPU  Name            TCC/WDDM | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
+            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
+            "|                               |                      |               MIG M. |\n",
+            "|===============================+======================+======================|\n",
+            "|   0  NVIDIA GeForce ... WDDM  | 00000000:01:00.0 Off |                  N/A |\n",
+            "| 31%   54C    P0    61W / 200W |    483MiB /  8192MiB |      0%      Default |\n",
+            "|                               |                      |                  N/A |\n",
+            "+-------------------------------+----------------------+----------------------+\n",
+            "                                                                               \n",
+            "+-----------------------------------------------------------------------------+\n",
+            "| Processes:                                                                  |\n",
+            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
+            "|        ID   ID                                                   Usage      |\n",
+            "|=============================================================================|\n",
+            "|    0   N/A  N/A      2520    C+G   ...lPanel\\SystemSettings.exe    N/A      |\n",
+            "|    0   N/A  N/A      3056    C+G   ...e\\PhoneExperienceHost.exe    N/A      |\n",
+            "|    0   N/A  N/A      7284    C+G   C:\\Windows\\explorer.exe         N/A      |\n",
+            "|    0   N/A  N/A      8860    C+G   ...me\\Application\\chrome.exe    N/A      |\n",
+            "|    0   N/A  N/A      8876    C+G   ...wekyb3d8bbwe\\Video.UI.exe    N/A      |\n",
+            "|    0   N/A  N/A      9024    C+G   ...5n1h2txyewy\\SearchApp.exe    N/A      |\n",
+            "|    0   N/A  N/A      9844    C+G   ...perience\\NVIDIA Share.exe    N/A      |\n",
+            "|    0   N/A  N/A     10504    C+G   ...cw5n1h2txyewy\\LockApp.exe    N/A      |\n",
+            "|    0   N/A  N/A     11820    C+G   ...2txyewy\\TextInputHost.exe    N/A      |\n",
+            "|    0   N/A  N/A     12652    C+G   ...sv5v3m8wq0b2\\ExcelApp.exe    N/A      |\n",
+            "|    0   N/A  N/A     12912    C+G   ...m Files\\iTunes\\iTunes.exe    N/A      |\n",
+            "|    0   N/A  N/A     13020    C+G   ...y\\ShellExperienceHost.exe    N/A      |\n",
+            "|    0   N/A  N/A     14760    C+G   ...8wekyb3d8bbwe\\Cortana.exe    N/A      |\n",
+            "|    0   N/A  N/A     15212    C+G   ...ck\\app-4.29.149\\slack.exe    N/A      |\n",
+            "|    0   N/A  N/A     16220    C+G   ...zpdnekdrzrea0\\Spotify.exe    N/A      |\n",
+            "|    0   N/A  N/A     16712    C+G   ...5n1h2txyewy\\SearchApp.exe    N/A      |\n",
+            "|    0   N/A  N/A     17004    C+G   ...lls\\wgc_renderer_host.exe    N/A      |\n",
+            "+-----------------------------------------------------------------------------+\n"
+          ]
+        }
+      ],
+      "source": [
+        "!nvidia-smi"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 33,
+      "metadata": {
+        "id": "Jsqm0FLOpyoS"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "Downloading: 100%|██████████| 1.54M/1.54M [00:00<00:00, 2.68MB/s]\n",
+            "Downloading: 100%|██████████| 1.21M/1.21M [00:00<00:00, 2.22MB/s]\n",
+            "Downloading: 100%|██████████| 574/574 [00:00<00:00, 287kB/s]\n",
+            "Downloading: 100%|██████████| 1.22k/1.22k [00:00<00:00, 612kB/s]\n",
+            "Downloading: 100%|██████████| 761/761 [00:00<00:00, 190kB/s]\n",
+            "Downloading: 100%|██████████| 1.61G/1.61G [02:47<00:00, 10.3MB/s]\n"
+          ]
+        }
+      ],
+      "source": [
+        "model_name_or_path = \"ai-forever/rugpt3medium_based_on_gpt2\"\n",
+        "tokenizer = GPT2Tokenizer.from_pretrained(model_name_or_path)\n",
+        "model = GPT2LMHeadModel.from_pretrained(model_name_or_path).to(DEVICE)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "w11Hq5Lvwcdu"
+      },
+      "source": [
+        "## Add special tokens"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 34,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "7ImM0Aq0dE8q",
+        "outputId": "c6fc436f-364e-461c-d908-cee9aa68eeeb"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "Embedding(50260, 1024)"
+            ]
+          },
+          "execution_count": 34,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "SPECIAL_TOKENS = {'bos_token':'<bos>','eos_token' :'<eos>', 'pad_token':'<pad>', 'sep_token': '<sep>'}\n",
+        "tokenizer.add_special_tokens(SPECIAL_TOKENS)\n",
+        "model.resize_token_embeddings(len(tokenizer))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DejAhD7dwjwX"
+      },
+      "source": [
+        "## Greate a Dataset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 35,
+      "metadata": {
+        "id": "vKsDqthMdxna"
+      },
+      "outputs": [],
+      "source": [
+        "torch.manual_seed(42)\n",
+        "\n",
+        "class myDataset(Dataset):\n",
+        "\n",
+        "  def __init__(self, data, tokenizer, gpt2_type=\"gpt2\", max_length=150):\n",
+        "    self.tokenizer = tokenizer # the gpt2 tokenizer we instantiated\n",
+        "    self.input_ids = []\n",
+        "    self.attn_masks = []\n",
+        "\n",
+        "    for i in data.index.to_list():\n",
+        "      title = data['title'][i]\n",
+        "      description = data['text'][i] if type(data['text'][i]) == str else ''\n",
+        "\n",
+        "      encodings_dict = tokenizer('<bos>'+ title + '<sep>' + description + '<eos>',\n",
+        "                                 truncation=True,\n",
+        "                                 max_length=max_length,\n",
+        "                                 padding=\"max_length\")\n",
+        "\n",
+        "      self.input_ids.append(torch.tensor(encodings_dict['input_ids']))\n",
+        "      self.attn_masks.append(torch.tensor(encodings_dict['attention_mask']))\n",
+        "\n",
+        "  def __len__(self):\n",
+        "    return len(self.input_ids)\n",
+        "\n",
+        "  def __getitem__(self, idx):\n",
+        "    return {\n",
+        "        'input_ids': self.input_ids[idx],\n",
+        "        'attn_masks': self.attn_masks[idx]\n",
+        "    }"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 36,
+      "metadata": {
+        "id": "607w_pnJMW5e"
+      },
+      "outputs": [],
+      "source": [
+        "train_dataset = myDataset(data, tokenizer)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 37,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "nK8r7CJr743J",
+        "outputId": "159787c7-cf79-4575-cb2c-91472c4cdebb"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "{'input_ids': tensor([   18, 39952,    16,   374, 27275,    47,    37,   840,  3215,  1904,\n",
+              "          1827,  5170,   428,  5554,  4426, 16686,  3250, 15366,   536, 36982,\n",
+              "          2480,  7768,    18,   365,  7570, 27275,    47,    37, 44030,  7872,\n",
+              "           519, 34308,  8482,   282,  1808,    18,   391,    42,    56,  3110,\n",
+              "            16,   374,  4866,  2147,  5304, 23322,   309, 12295,   519, 10367,\n",
+              "          8482,    18, 27275,    47,    37,  9727, 21691,  5512,  3672,  1134,\n",
+              "         16374,    30, 12604,  2009, 36272,   289, 24971,   473,    58,    56,\n",
+              "         31693,   934,   367,   827,   329,  1304,   912,  3772,   294, 13099,\n",
+              "           315,    17,   293,  4654,   660,  1941, 29216,    16,   502,  4347,\n",
+              "           942,  3836,   428,  6413, 37787, 36272,   289, 24971,    18,  2052,\n",
+              "          2050,  4921,  4266,    16,   374,   541,  3594,  7294,  2854, 32720,\n",
+              "         14813,  9971,  8253, 26282,    18, 16251,  5305, 29620,   434,  6566,\n",
+              "           482,  7385, 18460, 31495,   651,  1409, 14550,  2634,  4662,   651,\n",
+              "         10787,   282,   965, 14874, 10383,   933,   835,    16,  3118,  5456,\n",
+              "          4651,   309,  3504, 21499,   289, 31572,  5172,   416,  8408, 50258]),\n",
+              " 'attn_masks': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+              "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+              "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+              "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+              "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+              "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
+              "         1, 1, 1, 1, 1, 1])}"
+            ]
+          },
+          "execution_count": 37,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "train_dataset[10]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3H1pAm4Qwmvc"
+      },
+      "source": [
+        "## Add Datacollator"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 38,
+      "metadata": {
+        "id": "soPxrXgQpFZQ"
+      },
+      "outputs": [],
+      "source": [
+        "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kktpNyfnwrPY"
+      },
+      "source": [
+        "## Training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 39,
+      "metadata": {
+        "id": "iX2cSazzlm8E"
+      },
+      "outputs": [],
+      "source": [
+        "training_args = TrainingArguments(\n",
+        "    output_dir=f'{my_path}Checkouts', #The output directory\n",
+        "    overwrite_output_dir = True, #overwrite the content of the output directory\n",
+        "    num_train_epochs = 10, # number of training epochs\n",
+        "    per_device_train_batch_size = 3, # batch size for training\n",
+        "    per_device_eval_batch_size = 3,  # batch size for evaluation\n",
+        "    warmup_steps = 100,# number of warmup steps for learning rate scheduler\n",
+        "    gradient_accumulation_steps = 1, # to make \"virtual\" batch size larger\n",
+        "    save_steps = 3000\n",
+        "    )\n",
+        "\n",
+        "\n",
+        "trainer = Trainer(\n",
+        "    model=model,\n",
+        "    args=training_args,\n",
+        "    data_collator=data_collator,\n",
+        "    train_dataset=train_dataset,\n",
+        "    optimizers = (torch.optim.AdamW(model.parameters(),lr=1e-5),None) # Optimizer and lr scheduler\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 43,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 1000,
+          "referenced_widgets": [
+            "43a300cd4f3b4f26b5ef46bf4a7b4500",
+            "3cae9bd0f3e34a018f1c1770a65fb52f",
+            "d5607a93190249ce87179ea2456630f6",
+            "8bab8300f98b4df9a5ece6dafa8daba1",
+            "e88887e4031f4851ac6f5fd5e7f7b933",
+            "160d666ebc934016a0f30892832f37f0",
+            "2f49fd3cb2e2461eb562c403009c4acc",
+            "b510b835db64457d8e49ba3e965918de",
+            "15592d007be2409fa0dc6aaff0727554",
+            "1b63c50c01ca463487b94bd2ca3139f5",
+            "f1c2c1dc5cdd4e9ab3456d0e9bbb553d"
+          ]
+        },
+        "id": "OVtvq4sLq9EG",
+        "outputId": "88ea174a-5feb-4f4b-c4eb-0f34627b5cba"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "***** Running training *****\n",
+            "  Num examples = 10476\n",
+            "  Num Epochs = 10\n",
+            "  Instantaneous batch size per device = 3\n",
+            "  Total train batch size (w. parallel, distributed & accumulation) = 3\n",
+            "  Gradient Accumulation steps = 1\n",
+            "  Total optimization steps = 34920\n",
+            "  0%|          | 11/34920 [09:52<522:07:09, 53.84s/it]"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "\n",
+            "The following columns in the training set don't have a corresponding argument in `GPT2LMHeadModel.forward` and have been ignored: attn_masks. If attn_masks are not expected by `GPT2LMHeadModel.forward`,  you can safely ignore this message.\n"
+          ]
+        },
+        {
+          "ename": "KeyboardInterrupt",
+          "evalue": "",
+          "output_type": "error",
+          "traceback": [
+            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+            "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+            "Cell \u001b[1;32mIn[43], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
+            "File \u001b[1;32mc:\\GitHub\\TRAIN_GPT\\.conda\\Lib\\site-packages\\transformers\\trainer.py:1317\u001b[0m, in \u001b[0;36mtrain\u001b[1;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[0;32m   1314\u001b[0m     dtype \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mbfloat16 \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_cpu_amp \u001b[38;5;28;01melse\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mfloat32\n\u001b[0;32m   1315\u001b[0m     model \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mipex_optimize_model(model, training, dtype\u001b[38;5;241m=\u001b[39mdtype)\n\u001b[1;32m-> 1317\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_sagemaker_mp_enabled():\n\u001b[0;32m   1318\u001b[0m     \u001b[38;5;66;03m# Wrapping the base model twice in a DistributedModel will raise an error.\u001b[39;00m\n\u001b[0;32m   1319\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_wrapped, smp\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39mDistributedModel):\n\u001b[0;32m   1320\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_wrapped\n",
+            "File \u001b[1;32mc:\\GitHub\\TRAIN_GPT\\.conda\\Lib\\site-packages\\transformers\\trainer.py:1619\u001b[0m, in \u001b[0;36m_inner_training_loop\u001b[1;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[0;32m   1616\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlr_scheduler \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1617\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_created_lr_scheduler \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m-> 1619\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_deepspeed_enabled:\n\u001b[0;32m   1620\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptimizer, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlr_scheduler \u001b[38;5;241m=\u001b[39m deepspeed_init(\u001b[38;5;28mself\u001b[39m, num_training_steps\u001b[38;5;241m=\u001b[39mmax_steps)\n\u001b[0;32m   1622\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m delay_optimizer_creation:\n",
+            "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python311\\site-packages\\torch\\optim\\lr_scheduler.py:68\u001b[0m, in \u001b[0;36mLRScheduler.__init__.<locals>.with_counter.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     66\u001b[0m instance\u001b[38;5;241m.\u001b[39m_step_count \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m     67\u001b[0m wrapped \u001b[38;5;241m=\u001b[39m func\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__get__\u001b[39m(instance, \u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m---> 68\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwrapped\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+            "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python311\\site-packages\\torch\\optim\\optimizer.py:373\u001b[0m, in \u001b[0;36mOptimizer.profile_hook_step.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    368\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    369\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[0;32m    370\u001b[0m                 \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m must return None or a tuple of (new_args, new_kwargs), but got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresult\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    371\u001b[0m             )\n\u001b[1;32m--> 373\u001b[0m out \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    374\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_optimizer_step_code()\n\u001b[0;32m    376\u001b[0m \u001b[38;5;66;03m# call optimizer step post hooks\u001b[39;00m\n",
+            "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python311\\site-packages\\torch\\optim\\optimizer.py:76\u001b[0m, in \u001b[0;36m_use_grad_for_differentiable.<locals>._use_grad\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m     74\u001b[0m     torch\u001b[38;5;241m.\u001b[39mset_grad_enabled(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdefaults[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdifferentiable\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[0;32m     75\u001b[0m     torch\u001b[38;5;241m.\u001b[39m_dynamo\u001b[38;5;241m.\u001b[39mgraph_break()\n\u001b[1;32m---> 76\u001b[0m     ret \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     77\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m     78\u001b[0m     torch\u001b[38;5;241m.\u001b[39m_dynamo\u001b[38;5;241m.\u001b[39mgraph_break()\n",
+            "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python311\\site-packages\\torch\\optim\\adamw.py:184\u001b[0m, in \u001b[0;36mAdamW.step\u001b[1;34m(self, closure)\u001b[0m\n\u001b[0;32m    171\u001b[0m     beta1, beta2 \u001b[38;5;241m=\u001b[39m group[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbetas\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m    173\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_init_group(\n\u001b[0;32m    174\u001b[0m         group,\n\u001b[0;32m    175\u001b[0m         params_with_grad,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    181\u001b[0m         state_steps,\n\u001b[0;32m    182\u001b[0m     )\n\u001b[1;32m--> 184\u001b[0m     \u001b[43madamw\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    185\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparams_with_grad\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    186\u001b[0m \u001b[43m        \u001b[49m\u001b[43mgrads\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    187\u001b[0m \u001b[43m        \u001b[49m\u001b[43mexp_avgs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    188\u001b[0m \u001b[43m        \u001b[49m\u001b[43mexp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    189\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmax_exp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    190\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstate_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    191\u001b[0m \u001b[43m        \u001b[49m\u001b[43mamsgrad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mamsgrad\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    192\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbeta1\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbeta1\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    193\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbeta2\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbeta2\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    194\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    195\u001b[0m \u001b[43m        \u001b[49m\u001b[43mweight_decay\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mweight_decay\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    196\u001b[0m \u001b[43m        \u001b[49m\u001b[43meps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43meps\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    197\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmaximize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmaximize\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    198\u001b[0m \u001b[43m        \u001b[49m\u001b[43mforeach\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mforeach\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    199\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcapturable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcapturable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    200\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdifferentiable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdifferentiable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    201\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfused\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfused\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    202\u001b[0m \u001b[43m        \u001b[49m\u001b[43mgrad_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mgetattr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgrad_scale\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    203\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfound_inf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mgetattr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfound_inf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    204\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    206\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m loss\n",
+            "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python311\\site-packages\\torch\\optim\\adamw.py:335\u001b[0m, in \u001b[0;36madamw\u001b[1;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, foreach, capturable, differentiable, fused, grad_scale, found_inf, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize)\u001b[0m\n\u001b[0;32m    332\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    333\u001b[0m     func \u001b[38;5;241m=\u001b[39m _single_tensor_adamw\n\u001b[1;32m--> 335\u001b[0m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    336\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    337\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgrads\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    338\u001b[0m \u001b[43m    \u001b[49m\u001b[43mexp_avgs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    339\u001b[0m \u001b[43m    \u001b[49m\u001b[43mexp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    340\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmax_exp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    341\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstate_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    342\u001b[0m \u001b[43m    \u001b[49m\u001b[43mamsgrad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mamsgrad\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    343\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbeta1\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbeta1\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    344\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbeta2\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbeta2\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    345\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlr\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    346\u001b[0m \u001b[43m    \u001b[49m\u001b[43mweight_decay\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mweight_decay\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    347\u001b[0m \u001b[43m    \u001b[49m\u001b[43meps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    348\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmaximize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmaximize\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    349\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcapturable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcapturable\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    350\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdifferentiable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdifferentiable\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    351\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgrad_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgrad_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    352\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfound_inf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfound_inf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    353\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+            "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python311\\site-packages\\torch\\optim\\adamw.py:413\u001b[0m, in \u001b[0;36m_single_tensor_adamw\u001b[1;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, grad_scale, found_inf, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize, capturable, differentiable)\u001b[0m\n\u001b[0;32m    411\u001b[0m \u001b[38;5;66;03m# Decay the first and second moment running average coefficient\u001b[39;00m\n\u001b[0;32m    412\u001b[0m exp_avg\u001b[38;5;241m.\u001b[39mlerp_(grad, \u001b[38;5;241m1\u001b[39m \u001b[38;5;241m-\u001b[39m beta1)\n\u001b[1;32m--> 413\u001b[0m \u001b[43mexp_avg_sq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmul_\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbeta2\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39maddcmul_(grad, grad, value\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m \u001b[38;5;241m-\u001b[39m beta2)\n\u001b[0;32m    415\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m capturable \u001b[38;5;129;01mor\u001b[39;00m differentiable:\n\u001b[0;32m    416\u001b[0m     step \u001b[38;5;241m=\u001b[39m step_t\n",
+            "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
+          ]
+        }
+      ],
+      "source": [
+        "trainer.train()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7zL7BghWwtZH"
+      },
+      "source": [
+        "## Save models"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 41,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "R3CTFbR3lqOE",
+        "outputId": "1bfca296-21f8-4ede-d545-7f567d11f10c"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "Saving model checkpoint to content/model_with_summary\n",
+            "Configuration saved in content/model_with_summary\\config.json\n"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "Model weights saved in content/model_with_summary\\pytorch_model.bin\n"
+          ]
+        }
+      ],
+      "source": [
+        "trainer.save_model(f'{my_path}model_with_summary')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 42,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "-Dm5J3Ywieys",
+        "outputId": "e23711c3-67eb-48dc-dd07-6abc0ee090b6"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "Vocabulary path (content/tokenizer) should be a directory\n"
+          ]
+        }
+      ],
+      "source": [
+        "tokenizer.save_vocabulary(f'{my_path}tokenizer')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xxkC-HnujEQu"
+      },
+      "source": [
+        "#Generation"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pJtoWx-WMY6W"
+      },
+      "outputs": [],
+      "source": [
+        "tokenizer = GPT2Tokenizer.from_pretrained(f'{my_path}tokenizer')\n",
+        "model = GPT2LMHeadModel.from_pretrained(f'{my_path}model_with_summary').to(DEVICE)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "TKzQfU7wleKK",
+        "outputId": "92e40eb9-00c8-40a5-f649-01d87954a8ef"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "3"
+            ]
+          },
+          "execution_count": 11,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "SPECIAL_TOKENS = {'bos_token':'<bos>','eos_token' :'<eos>', 'pad_token':'<pad>', 'sep_token': '<sep>'}\n",
+        "tokenizer.add_special_tokens(SPECIAL_TOKENS)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kofX6i57RcLd"
+      },
+      "outputs": [],
+      "source": [
+        "class KeywordsStoppingCriteria(StoppingCriteria):\n",
+        "    def __init__(self, keywords_ids:list):\n",
+        "      self.keywords = keywords_ids\n",
+        "\n",
+        "    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:\n",
+        "      if input_ids[0][-1] in self.keywords:\n",
+        "        print(input_ids)\n",
+        "        return True\n",
+        "      return False"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "npykPtVhRcLd"
+      },
+      "outputs": [],
+      "source": [
+        "stop_criteria = KeywordsStoppingCriteria(tokenizer.encode(tokenizer.eos_token, return_tensors=\"pt\").to(DEVICE))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-TVOwDiNmwqw"
+      },
+      "source": [
+        "## Задай свое начало предложения:\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "upBkDzPkFtD0",
+        "outputId": "d248d2e0-4476-46f7-80a7-61963738ef67"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Введите начало заголовка или просто нажмите Enter:\n"
+          ]
+        }
+      ],
+      "source": [
+        "inp = input('Введите начало заголовка или просто нажмите Enter:')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VLaeBnu1RcLe"
+      },
+      "outputs": [],
+      "source": [
+        "inp = inp if len(inp) > 0 else tokenizer.bos_token #токен начала предложения\n",
+        "input_ids = tokenizer.encode(inp, return_tensors=\"pt\").to(DEVICE)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "xclcUo5cRcLf",
+        "outputId": "05a741f0-6d9a-496f-ec28-c54c5e0d9807"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "Setting `pad_token_id` to `eos_token_id`:50258 for open-end generation.\n"
+          ]
+        },
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "<bos> Ученые впервые нашли венерические заболевания с помощью генной инженерии <sep> Ученые выяснили, что венерические заболевания с помощью генной терапии передаются исключительно половым путем. Это открытие может лечь в основу разработки метода контрацепции, основанного на генной терапии.&nbsp; <eos>\n"
+          ]
+        }
+      ],
+      "source": [
+        "with torch.no_grad():\n",
+        "      out = model.generate(input_ids,\n",
+        "                          do_sample=True,\n",
+        "                          num_beams=3,\n",
+        "                          temperature=2.0,\n",
+        "                          top_p=0.9,\n",
+        "                          max_length = 200,\n",
+        "                          stopping_criteria=StoppingCriteriaList([stop_criteria]),\n",
+        "                          eos_token_id=tokenizer.eos_token_id,\n",
+        "                          bos_token_id=tokenizer.bos_token_id,\n",
+        "                          ).to(DEVICE)\n",
+        "print(tokenizer.batch_decode(out, skip_special_tokens=False)[0])"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "include_colab_link": true,
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.11.5"
+    },
+    "widgets": {
+      "application/vnd.jupyter.widget-state+json": {
+        "15592d007be2409fa0dc6aaff0727554": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "160d666ebc934016a0f30892832f37f0": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "1b63c50c01ca463487b94bd2ca3139f5": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "2f49fd3cb2e2461eb562c403009c4acc": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "3cae9bd0f3e34a018f1c1770a65fb52f": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_160d666ebc934016a0f30892832f37f0",
+            "placeholder": "​",
+            "style": "IPY_MODEL_2f49fd3cb2e2461eb562c403009c4acc",
+            "value": "Skipping the first batches: 100%"
+          }
+        },
+        "43a300cd4f3b4f26b5ef46bf4a7b4500": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_3cae9bd0f3e34a018f1c1770a65fb52f",
+              "IPY_MODEL_d5607a93190249ce87179ea2456630f6",
+              "IPY_MODEL_8bab8300f98b4df9a5ece6dafa8daba1"
+            ],
+            "layout": "IPY_MODEL_e88887e4031f4851ac6f5fd5e7f7b933"
+          }
+        },
+        "8bab8300f98b4df9a5ece6dafa8daba1": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_1b63c50c01ca463487b94bd2ca3139f5",
+            "placeholder": "​",
+            "style": "IPY_MODEL_f1c2c1dc5cdd4e9ab3456d0e9bbb553d",
+            "value": " 48/48 [00:00&lt;00:00, 962.01it/s]"
+          }
+        },
+        "b510b835db64457d8e49ba3e965918de": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "d5607a93190249ce87179ea2456630f6": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_b510b835db64457d8e49ba3e965918de",
+            "max": 48,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_15592d007be2409fa0dc6aaff0727554",
+            "value": 48
+          }
+        },
+        "e88887e4031f4851ac6f5fd5e7f7b933": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "f1c2c1dc5cdd4e9ab3456d0e9bbb553d": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        }
+      }
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

main.py

@@ -0,0 +1,19 @@
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+import torch
+DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+# Для наглядности будем работать с русскоязычной GPT от Сбера.
+# Ниже команды для загрузки и инициализации модели и токенизатора.
+model_name_or_path = "ai-forever/rugpt3medium_based_on_gpt2"
+tokenizer = GPT2Tokenizer.from_pretrained(model_name_or_path)
+model = GPT2LMHeadModel.from_pretrained(model_name_or_path).to(DEVICE)
+
+
+# prompt engineering for QA
+text = "Вопрос: 'Сколько будет 2+2?'\nОтвет:" 
+input_ids = tokenizer.encode(text, return_tensors="pt").to(DEVICE)
+out = model.generate(input_ids, do_sample=False) 
+
+generated_text = list(map(tokenizer.decode, out))[0]
+print(generated_text)
+

test.ipynb

@@ -0,0 +1,491 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\addmi\\AppData\\Roaming\\Python\\Python311\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import GPT2LMHeadModel, GPT2Config, GPT2Tokenizer, TextDataset, DataCollatorForLanguageModeling, Trainer, TrainingArguments\n",
+    "\n",
+    "model_name_or_path = \"ai-forever/rugpt3small_based_on_gpt2\"\n",
+    "tokenizer = GPT2Tokenizer.from_pretrained(model_name_or_path)\n",
+    "model = GPT2LMHeadModel.from_pretrained(model_name_or_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "('4.10.0', '0.20.1')"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import accelerate\n",
+    "import transformers\n",
+    "\n",
+    "transformers.__version__, accelerate.__version__"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = \"\"\"Трогательным ножичком пытать свою плоть\n",
+    "Трогательным ножичком пытать свою плоть\n",
+    "До крови прищемить добровольные пальцы\n",
+    "Отважно смакуя леденцы на палочке\n",
+    "Целеустремлённо набивать карманы\n",
+    "Мёртвыми мышатами,живыми хуями\n",
+    "Шоколадными конфетами \n",
+    "И нерукотворными пиздюлями\n",
+    "На патриархальной свалке устаревших понятий\n",
+    "Использованных образов и вежливых слов\n",
+    "Покончив с собою,уничтожить весь мир\n",
+    "ПОКОНЧИВ С СОБОЮ-УНИЧТОЖИТЬ ВЕСЬ МИР!!\n",
+    "\n",
+    "        ...вечность пахнет нефтью\n",
+    "        ...вечность пахнет нефтью\n",
+    "\n",
+    "Словно иней,сердобольный смех\n",
+    "Словно иней,сердобольный смех\n",
+    "Словно иней,сердобольный смех\n",
+    "Славно валится на...\n",
+    "                 на...\n",
+    "                 на РУССКОЕ ПОЛЕ ЭКСПЕРИМЕНТОВ\n",
+    "\n",
+    "География подлости\n",
+    "Орфография ненависти\n",
+    "Апология невежества\n",
+    "Мифология оптимизма\n",
+    "Законы гаубицы благонравия\n",
+    "Знатное пиршество благоразумия\n",
+    "Устами ребенка глаголет яма\n",
+    "Устами ребенка глаголет пуля\n",
+    "\n",
+    "         ...вечность пахнет нефтью\n",
+    "         ...вечность пахнет нефтью\n",
+    "\n",
+    "Мастерство быть излишним,подобно мне\n",
+    "Мастерство быть любимым,подобно петле\n",
+    "Мастерство быть глобальным,как печёное яблоко\n",
+    "Искусство вовремя уйти в сторонку\n",
+    "   Искусство быть посторонним\n",
+    "   Искусство быть посторонним\n",
+    "Новейшее средство для очистки духовок\n",
+    "От задохнувшихся по собственной воле\n",
+    "Новейшее средство для очистки верёвок\n",
+    "От скверного запаха немытых  шей\n",
+    "   Новейшее средство находить виновных\n",
+    "   Новейшее средство находить виновных\n",
+    "\n",
+    "         РУССКОЕ ПОЛЕ ЭКСПЕРИМЕНТОВ\n",
+    "\n",
+    "За открывшейся дверью — пустота\n",
+    "Это значит,что кто-то пришёл за тобой\n",
+    "Это значит,что теперь ты кому-то                         \n",
+    "Понадобился\n",
+    "\n",
+    "А снег всё идёт,а снег всё идёт\n",
+    "Русское поле источает снег\n",
+    "\n",
+    "Иных хоронили в упаковке глазёнок\n",
+    "Иных хоронили в упаковке газет\n",
+    "\n",
+    "А то,что на бойне умертвили бычка \n",
+    "На то всеобщая радость,всеобщая гордость\n",
+    "Всеобщая ненависть,всеобщая воля\n",
+    "Всеобщая воля да всеобщая старость\n",
+    "\n",
+    "Набить до отказа собой могилу \n",
+    "Это значит наследовать землю\n",
+    "Что же такое наследовать землю\n",
+    "Это значит исчерпать терпение\n",
+    "   Что и требовалось доказать\n",
+    "   Что и требовалось доказать\n",
+    "\n",
+    "В дверной глазок — в замочную щель \n",
+    "Гениальные мыслишки — мировые войнушки\n",
+    "Неофициальные пупы земли\n",
+    "Эмалированные части головных систем\n",
+    "Инстинктивные добровольцы\n",
+    "Во имя вселенной и хлебной корочки\n",
+    "Люди с большой буквы\n",
+    "Слово «люди» пишется с большой буквы\n",
+    "\n",
+    "Свастика веры стянула лица\n",
+    "Вавилонская азбука налипла на пальцах\n",
+    "Исторически оправданный метод\n",
+    "Пожирания сырой земли\n",
+    "   Это ли не то,что нам надо?!\n",
+    "   Это ли не то,что нам надо?!\n",
+    "\n",
+    "А поутру они неизбежно проснулись\n",
+    "Не простудились — не замарались\n",
+    "Называли вещи своими именами\n",
+    "Сеяли доброе,разумное,вечное\n",
+    "Всё посеяли,всё назвали\n",
+    "Кушать подано — честь по чести\n",
+    "На первое были плоды просвещения\n",
+    "А на второе — кровавые мальчики\n",
+    "\n",
+    "Орденоносный господь победоносного мира\n",
+    "Заслуженный господь краснознамённого страха\n",
+    "Праведный праздник для правильных граждан\n",
+    "Отточенный серп для созревших колосьев\n",
+    "\n",
+    "Яма как принцип движения к Солнцу\n",
+    "Кашу слезами не испортишь нет\n",
+    "\n",
+    "Полные сани девичьим срамом\n",
+    "Полные простыни ребячьим смрадом\n",
+    "Девичьи глазки,кукушкины слёзки\n",
+    "А так же всякие иные предметы\n",
+    "\n",
+    "Так кто погиб в генеральном сраженьи\n",
+    "Кто погиб в гениальном поражении\n",
+    "За полную чашку жалости\n",
+    "В Сталинградской битве озверевшей похоти?\n",
+    "\n",
+    "Самолёт усмехнулся вдребезги\n",
+    "В бугорок обетованной земли\n",
+    "Самолёт усмехнулся вдребезги\n",
+    "В бугорок обетованной землицы\n",
+    "\n",
+    "А свою любовь я собственноручно\n",
+    "Освободил от дальнейших неизбежных огорчений\n",
+    "Подманил ее пряником\n",
+    "Подманил ее пряником  \n",
+    "Изнасиловал пьяным жестоким ботинком\n",
+    "И повесил на облачке,словно ребёнок\n",
+    "        СВОЮ НЕЛЮБИМУЮ КУКЛУ\n",
+    "        СВОЮ НЕЛЮБИМУЮ КУКЛУ\n",
+    "        СВОЮ НЕЛЮБИМУЮ КУКЛУ\n",
+    "        СВОЮ НЕЛЮБИМУЮ КУКЛУ\n",
+    "\n",
+    "Словно иней,сердобольный смех \n",
+    "Словно иней,сердобольный смех \n",
+    "Словно иней,сердобольный смех \n",
+    "Словно иней,сердобольный смех \n",
+    "Славно валиться на...\n",
+    "                 на...\n",
+    "                 на РУССКОЕ ПОЛЕ ЭКСПЕРИМЕНТОВ\n",
+    " \n",
+    "       Вечность пахнет нефтью\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\GitHub\\TRAIN_GPT\\.conda\\Lib\\site-packages\\transformers\\data\\datasets\\language_modeling.py:54: FutureWarning: This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/pytorch/language-modeling/run_mlm.py\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Сохраним обучающие данные в .txt файл\n",
+    "train_path = 'GROB.txt'\n",
+    "'''\n",
+    "with open(train_path, \"w\") as f:\n",
+    "    f.write(text)'''\n",
+    "\n",
+    "# Создание датасета\n",
+    "train_dataset = TextDataset(tokenizer=tokenizer, file_path=train_path, block_size=64)\n",
+    "\n",
+    "# Создание даталодера (нарезает текст на оптимальные по длине куски)\n",
+    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "PyTorch: setting up devices\n",
+      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Подготовка параметров fine-tuning\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"./gpt2-finetuned\",\n",
+    "    overwrite_output_dir=True,\n",
+    "    num_train_epochs=10,  # Количество эпох обучения\n",
+    "    per_device_train_batch_size=32,\n",
+    "    per_device_eval_batch_size=32,\n",
+    "    save_steps=10_000,  # Сохранять модель каждые 10000 шагов\n",
+    "    save_total_limit=2,\n",
+    "    gradient_accumulation_steps=16,\n",
+    ")\n",
+    "\n",
+    "# Создание объекта Trainer\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    data_collator=data_collator,\n",
+    "    train_dataset=train_dataset,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "***** Running training *****\n",
+      "  Num examples = 4477\n",
+      "  Num Epochs = 10\n",
+      "  Instantaneous batch size per device = 32\n",
+      "  Total train batch size (w. parallel, distributed & accumulation) = 512\n",
+      "  Gradient Accumulation steps = 16\n",
+      "  Total optimization steps = 80\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": []
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[7], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Запуск fine-tuning\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[1;32mc:\\GitHub\\TRAIN_GPT\\.conda\\Lib\\site-packages\\transformers\\trainer.py:1284\u001b[0m, in \u001b[0;36mTrainer.train\u001b[1;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[0;32m   1282\u001b[0m         tr_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining_step(model, inputs)\n\u001b[0;32m   1283\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1284\u001b[0m     tr_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtraining_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1285\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcurrent_flos \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mfloat\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfloating_point_ops(inputs))\n\u001b[0;32m   1287\u001b[0m \u001b[38;5;66;03m# Optimizer step for deepspeed must be called on every step regardless of the value of gradient_accumulation_steps\u001b[39;00m\n",
+      "File \u001b[1;32mc:\\GitHub\\TRAIN_GPT\\.conda\\Lib\\site-packages\\transformers\\trainer.py:1807\u001b[0m, in \u001b[0;36mTrainer.training_step\u001b[1;34m(self, model, inputs)\u001b[0m\n\u001b[0;32m   1805\u001b[0m     loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdeepspeed\u001b[38;5;241m.\u001b[39mbackward(loss)\n\u001b[0;32m   1806\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1807\u001b[0m     \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1809\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m loss\u001b[38;5;241m.\u001b[39mdetach()\n",
+      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python311\\site-packages\\torch\\_tensor.py:492\u001b[0m, in \u001b[0;36mTensor.backward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    482\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    483\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[0;32m    484\u001b[0m         Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[0;32m    485\u001b[0m         (\u001b[38;5;28mself\u001b[39m,),\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    490\u001b[0m         inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[0;32m    491\u001b[0m     )\n\u001b[1;32m--> 492\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    493\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[0;32m    494\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python311\\site-packages\\torch\\autograd\\__init__.py:251\u001b[0m, in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    246\u001b[0m     retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[0;32m    248\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[0;32m    249\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[0;32m    250\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[1;32m--> 251\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[0;32m    252\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    253\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    254\u001b[0m \u001b[43m    \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    255\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    256\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    257\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[0;32m    258\u001b[0m \u001b[43m    \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[0;32m    259\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "# Запуск fine-tuning\n",
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Configuration saved in ./gpt2-finetuned\\config.json\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Model weights saved in ./gpt2-finetuned\\pytorch_model.bin\n",
+      "tokenizer config file saved in ./gpt2-finetuned\\tokenizer_config.json\n",
+      "Special tokens file saved in ./gpt2-finetuned\\special_tokens_map.json\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "('./gpt2-finetuned\\\\tokenizer_config.json',\n",
+       " './gpt2-finetuned\\\\special_tokens_map.json',\n",
+       " './gpt2-finetuned\\\\vocab.json',\n",
+       " './gpt2-finetuned\\\\merges.txt',\n",
+       " './gpt2-finetuned\\\\added_tokens.json')"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Сохранение обученной модели\n",
+    "model.save_pretrained(\"./gpt2-finetuned\")\n",
+    "tokenizer.save_pretrained(\"./gpt2-finetuned\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Сгенерированный вопрос: Новый год к нам мчится\n",
+      "Гениальные мыслишки лезут в замочную щель\n",
+      "Неофициальные пупы земли\n",
+      "Эмалированные части головных систем\n",
+      "Инстинктивные добровольцы\n",
+      "Во имя вселенной и хлебной корочки\n",
+      "Люди с большой буквы\n",
+      "Слово «люди» пишется со значком «пожиратель».\n",
+      "\n",
+      "Яма как принцип движения к Солнцу\n",
+      "Кашу слезами не испортишь нет\n",
+      "   Это ли не то,что нам надо?!\n",
+      "                 на РУССКОЕ ПОЛЕ ЭКСПЕРИМЕНТОВ\n",
+      "Новейшее средство для очистки духовок\n",
+      "От задохнувшихся по собственной воле\n",
+      "Сеяли доброе,разумное,вечное\n",
+      "Всё посеяли,всё назвали\n",
+      "Кушать подано — честь по чести\n",
+      "На первое были плоды просвещения\n",
+      "А на второе — кровавые мальчики\n",
+      " \n",
+      "\n",
+      "Орденоносный праздник для правильных граждан\n",
+      "За полную чашку жалости\n",
+      "В дверной глазок не загляни\n",
+      "Полные простыни надежды\n",
+      "Девичьи глазки,кукушкины слёзки\n",
+      "Иные погибали в генеральном сраженьи\n",
+      "Штрафбаты,войны,колониальные реформы\n",
+      "Мифология оптимизма\n",
+      "Законы гаубицы благонравия\n",
+      "Знатное пиршество благоразумия\n",
+      "Устами ребенка глаголет пуля\n",
+      "        ...вечность пахнет нефтью\n",
+      "        СВОЮ НЕЛЮБИМУЮ КУ\n"
+     ]
+    },
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mПри выполнении кода в текущей ячейке или предыдущей ячейке ядро аварийно завершило работу. Проверьте код в ячейках, чтобы определить возможную причину сбоя. Щелкните <a href=\"https://aka.ms/vscodeJupyterKernelCrash\">здесь</a> для получения дополнительных сведений. Подробнее см. в <a href='command:jupyter.viewOutput'>журнале Jupyter</a>."
+     ]
+    }
+   ],
+   "source": [
+    "# Задаем контекст для генерации вопроса\n",
+    "context = \"Новый год к нам мчится\"\n",
+    "input_ids = tokenizer.encode(context, return_tensors=\"pt\")\n",
+    "\n",
+    "# Генерация текста (вопроса)\n",
+    "output = model.generate(input_ids, max_length=300, num_beams=5, no_repeat_ngram_size=2, top_k=50, top_p=0.95, temperature=0.7)\n",
+    "\n",
+    "# Декодирование сгенерированного текста\n",
+    "question = tokenizer.decode(output[0], skip_special_tokens=True)\n",
+    "\n",
+    "print(\"Сгенерированный вопрос:\", question)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Трогательным ножечком пытать свою плоть!\n",
+      "Трогательным ножечком пытать свою плоть!\n",
+      "До крови прищемить добровольные пальцы\n",
+      "Отважно смакуя леденцы на палочке\n",
+      "Целеустремлённо набивать карманы\n",
+      "Мёртвыми мышатами,живыми хуями\n",
+      "Шрамом на пальцах,живыми хуями\n",
+      "Шрамом на пальцах,живыми хуями\n",
+      "\n",
+      "А свою любовь я собственноручно\n",
+      "Освободил от дальнейших неизбеж\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "# Пример вероя��ностного сэмплирвоания с ограничением\n",
+    "text = \"Трогательным ножечком пытать свою плоть!\\n\"\n",
+    "input_ids = tokenizer.encode(text, return_tensors=\"pt\")\n",
+    "model.eval()\n",
+    "with torch.no_grad():\n",
+    "    out = model.generate(input_ids,\n",
+    "                        do_sample=True,\n",
+    "                        num_beams=2,\n",
+    "                        temperature=1.5,\n",
+    "                        top_p=0.9,\n",
+    "                        max_length=100,\n",
+    "                        )\n",
+    "\n",
+    "generated_text = list(map(tokenizer.decode, out))[0]\n",
+    "print()\n",
+    "print(generated_text)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

train_dataset.txt

@@ -0,0 +1,141 @@
+Трогательным ножичком пытать свою плоть
+Трогательным ножичком пытать свою плоть
+До крови прищемить добровольные пальцы
+Отважно смакуя леденцы на палочке
+Целеустремлённо набивать карманы
+Мёртвыми мышатами,живыми хуями
+Шоколадными конфетами 
+И нерукотворными пиздюлями
+На патриархальной свалке устаревших понятий
+Использованных образов и вежливых слов
+Покончив с собою,уничтожить весь мир
+ПОКОНЧИВ С СОБОЮ-УНИЧТОЖИТЬ ВЕСЬ МИР!!
+
+        ...вечность пахнет нефтью
+        ...вечность пахнет нефтью
+
+Словно иней,сердобольный смех
+Словно иней,сердобольный смех
+Словно иней,сердобольный смех
+Славно валится на...
+                 на...
+                 на РУССКОЕ ПОЛЕ ЭКСПЕРИМЕНТОВ
+
+География подлости
+Орфография ненависти
+Апология невежества
+Мифология оптимизма
+Законы гаубицы благонравия
+Знатное пиршество благоразумия
+Устами ребенка глаголет яма
+Устами ребенка глаголет пуля
+
+         ...вечность пахнет нефтью
+         ...вечность пахнет нефтью
+
+Мастерство быть излишним,подобно мне
+Мастерство быть любимым,подобно петле
+Мастерство быть глобальным,как печёное яблоко
+Искусство вовремя уйти в сторонку
+   Искусство быть посторонним
+   Искусство быть посторонним
+Новейшее средство для очистки духовок
+От задохнувшихся по собственной воле
+Новейшее средство для очистки верёвок
+От скверного запаха немытых  шей
+   Новейшее средство находить виновных
+   Новейшее средство находить виновных
+
+         РУССКОЕ ПОЛЕ ЭКСПЕРИМЕНТОВ
+
+За открывшейся дверью — пустота
+Это значит,что кто-то пришёл за тобой
+Это значит,что теперь ты кому-то                         
+Понадобился
+
+А снег всё идёт,а снег всё идёт
+Русское поле источает снег
+
+Иных хоронили в упаковке глазёнок
+Иных хоронили в упаковке газет
+
+А то,что на бойне умертвили бычка 
+На то всеобщая радость,всеобщая гордость
+Всеобщая ненависть,всеобщая воля
+Всеобщая воля да всеобщая старость
+
+Набить до отказа собой могилу 
+Это значит наследовать землю
+Что же такое наследовать землю
+Это значит исчерпать терпение
+   Что и требовалось доказать
+   Что и требовалось доказать
+
+В дверной глазок — в замочную щель 
+Гениальные мыслишки — мировые войнушки
+Неофициальные пупы земли
+Эмалированные части головных систем
+Инстинктивные добровольцы
+Во имя вселенной и хлебной корочки
+Люди с большой буквы
+Слово «люди» пишется с большой буквы
+
+Свастика веры стянула лица
+Вавилонская азбука налипла на пальцах
+Исторически оправданный метод
+Пожирания сырой земли
+   Это ли не то,что нам надо?!
+   Это ли не то,что нам надо?!
+
+А поутру они неизбежно проснулись
+Не простудились — не замарались
+Называли вещи своими именами
+Сеяли доброе,разумное,вечное
+Всё посеяли,всё назвали
+Кушать подано — честь по чести
+На первое были плоды просвещения
+А на второе — кровавые мальчики
+
+Орденоносный господь победоносного мира
+Заслуженный господь краснознамённого страха
+Праведный праздник для правильных граждан
+Отточенный серп для созревших колосьев
+
+Яма как принцип движения к Солнцу
+Кашу слезами не испортишь нет
+
+Полные сани девичьим срамом
+Полные простыни ребячьим смрадом
+Девичьи глазки,кукушкины слёзки
+А так же всякие иные предметы
+
+Так кто погиб в генеральном сраженьи
+Кто погиб в гениальном поражении
+За полную чашку жалости
+В Сталинградской битве озверевшей похоти?
+
+Самолёт усмехнулся вдребезги
+В бугорок обетованной земли
+Самолёт усмехнулся вдребезги
+В бугорок обетованной землицы
+
+А свою любовь я собственноручно
+Освободил от дальнейших неизбежных огорчений
+Подманил ее пряником
+Подманил ее пряником  
+Изнасиловал пьяным жестоким ботинком
+И повесил на облачке,словно ребёнок
+        СВОЮ НЕЛЮБИМУЮ КУКЛУ
+        СВОЮ НЕЛЮБИМУЮ КУКЛУ
+        СВОЮ НЕЛЮБИМУЮ КУКЛУ
+        СВОЮ НЕЛЮБИМУЮ КУКЛУ
+
+Словно иней,сердобольный смех 
+Словно иней,сердобольный смех 
+Словно иней,сердобольный смех 
+Словно иней,сердобольный смех 
+Славно валиться на...
+                 на...
+                 на РУССКОЕ ПОЛЕ ЭКСПЕРИМЕНТОВ
+ 
+       Вечность пахнет нефтью

unit_all_md_to_one_file.ipynb

@@ -0,0 +1,77 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Объединенный файл сохранен в 'GROB.txt'.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "\n",
+    "def combine_md_to_txt(input_folder, output_file):\n",
+    "    # Проверяем существование каталога\n",
+    "    if not os.path.exists(input_folder):\n",
+    "        print(f\"Указанный каталог '{input_folder}' не существует.\")\n",
+    "        return\n",
+    "\n",
+    "    # Получаем список файлов с расширением .md\n",
+    "    md_files = [f for f in os.listdir(input_folder) if f.endswith('.md')]\n",
+    "\n",
+    "    # Проверяем, есть ли файлы для объединения\n",
+    "    if not md_files:\n",
+    "        print(f\"В каталоге '{input_folder}' нет файлов с расширением .md для объединения.\")\n",
+    "        return\n",
+    "\n",
+    "    # Объединяем содержимое файлов\n",
+    "    combined_content = \"\"\n",
+    "    for md_file in md_files:\n",
+    "        file_path = os.path.join(input_folder, md_file)\n",
+    "        with open(file_path, 'r', encoding='utf-8') as file:\n",
+    "            #combined_content += f\"# {md_file}\\n\\n\"  # Добавляем заголовок с именем файла\n",
+    "            combined_content += file.read() + \"\\n\\n\"  # Добавляем содержимое файла\n",
+    "\n",
+    "    # Записываем объединенный контент в файл .txt\n",
+    "    with open(output_file, 'w', encoding='utf-8') as output:\n",
+    "        output.write(combined_content)\n",
+    "\n",
+    "    print(f\"Объединенный файл сохранен в '{output_file}'.\")\n",
+    "\n",
+    "# Укажите путь к каталогу с файлами .md и имя выходного файла .txt\n",
+    "input_folder_path = 'GROB'\n",
+    "output_file_path = 'GROB.txt'\n",
+    "\n",
+    "# Вызываем функцию для объединения файлов\n",
+    "combine_md_to_txt(input_folder_path, output_file_path)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}