Upload test.ipynb with huggingface_hub

test.ipynb

@@ -0,0 +1,414 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "execution": {
+     "iopub.execute_input": "2023-03-24T15:35:18.881502Z",
+     "iopub.status.busy": "2023-03-24T15:35:18.880789Z",
+     "iopub.status.idle": "2023-03-24T15:35:22.115494Z",
+     "shell.execute_reply": "2023-03-24T15:35:22.114665Z",
+     "shell.execute_reply.started": "2023-03-24T15:35:18.881474Z"
+    },
+    "id": "Ef7vm_ycPCxw",
+    "outputId": "95289963-04fd-4911-aece-c4ea17b40cca"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: transformers in /usr/local/lib/python3.9/dist-packages (4.21.3)\n",
+      "Requirement already satisfied: datasets in /usr/local/lib/python3.9/dist-packages (2.4.0)\n",
+      "Requirement already satisfied: torch in /usr/local/lib/python3.9/dist-packages (1.12.1+cu116)\n",
+      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.12.0)\n",
+      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n",
+      "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.28.2)\n",
+      "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.9.0)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (5.4.1)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
+      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.23.4)\n",
+      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.64.1)\n",
+      "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.12.1)\n",
+      "Requirement already satisfied: multiprocess in /usr/local/lib/python3.9/dist-packages (from datasets) (0.70.13)\n",
+      "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.5.0)\n",
+      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.1.0)\n",
+      "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from datasets) (3.8.3)\n",
+      "Requirement already satisfied: xxhash in /usr/local/lib/python3.9/dist-packages (from datasets) (3.2.0)\n",
+      "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.18.0)\n",
+      "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (10.0.1)\n",
+      "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.3.5.1)\n",
+      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch) (4.4.0)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.1)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (6.0.4)\n",
+      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.1.1)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.3)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (18.2.0)\n",
+      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (4.0.2)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.8.2)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers) (2019.11.28)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers) (2.8)\n",
+      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.14)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n",
+      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.14.0)\n",
+      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
+      "\u001b[0m"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install transformers datasets torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "execution": {
+     "iopub.execute_input": "2023-03-24T15:39:46.744379Z",
+     "iopub.status.busy": "2023-03-24T15:39:46.744112Z",
+     "iopub.status.idle": "2023-03-24T15:49:07.678229Z",
+     "shell.execute_reply": "2023-03-24T15:49:07.677294Z",
+     "shell.execute_reply.started": "2023-03-24T15:39:46.744360Z"
+    },
+    "id": "Kss9x-rwPFTS",
+    "outputId": "6297d81a-6053-4cb3-b5e2-cd694f95b0e6"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on cuda\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using cuda_amp half precision backend\n",
+      "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
+      "  warnings.warn(\n",
+      "***** Running training *****\n",
+      "  Num examples = 43\n",
+      "  Num Epochs = 3\n",
+      "  Instantaneous batch size per device = 4\n",
+      "  Total train batch size (w. parallel, distributed & accumulation) = 4\n",
+      "  Gradient Accumulation steps = 1\n",
+      "  Total optimization steps = 33\n",
+      "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "  ········································\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "wandb version 0.14.0 is available!  To upgrade, please run:\n",
+       " $ pip install wandb --upgrade"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Tracking run with wandb version 0.13.4"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Run data is saved locally in <code>/notebooks/wandb/run-20230324_154130-vctszt1m</code>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Syncing run <strong><a href=\"https://wandb.ai/subbytemp/huggingface/runs/vctszt1m\" target=\"_blank\">./output</a></strong> to <a href=\"https://wandb.ai/subbytemp/huggingface\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='33' max='33' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [33/33 07:17, Epoch 3/3]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Epoch</th>\n",
+       "      <th>Training Loss</th>\n",
+       "      <th>Validation Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>1.520941</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>1.267309</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>No log</td>\n",
+       "      <td>1.192186</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "***** Running Evaluation *****\n",
+      "  Num examples = 8\n",
+      "  Batch size = 4\n",
+      "Saving model checkpoint to ./output/checkpoint-11\n",
+      "Configuration saved in ./output/checkpoint-11/config.json\n",
+      "The model is bigger than the maximum size per checkpoint (10GB) and is going to be split in 2 checkpoint shards. You can find where each parameters has been saved in the index located at ./output/checkpoint-11/pytorch_model.bin.index.json.\n",
+      "***** Running Evaluation *****\n",
+      "  Num examples = 8\n",
+      "  Batch size = 4\n",
+      "Saving model checkpoint to ./output/checkpoint-22\n",
+      "Configuration saved in ./output/checkpoint-22/config.json\n",
+      "The model is bigger than the maximum size per checkpoint (10GB) and is going to be split in 2 checkpoint shards. You can find where each parameters has been saved in the index located at ./output/checkpoint-22/pytorch_model.bin.index.json.\n",
+      "***** Running Evaluation *****\n",
+      "  Num examples = 8\n",
+      "  Batch size = 4\n",
+      "Saving model checkpoint to ./output/checkpoint-33\n",
+      "Configuration saved in ./output/checkpoint-33/config.json\n",
+      "The model is bigger than the maximum size per checkpoint (10GB) and is going to be split in 2 checkpoint shards. You can find where each parameters has been saved in the index located at ./output/checkpoint-33/pytorch_model.bin.index.json.\n",
+      "\n",
+      "\n",
+      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
+      "\n",
+      "\n",
+      "Configuration saved in ./model/config.json\n",
+      "tokenizer config file saved in ./model/tokenizer_config.json\n",
+      "Special tokens file saved in ./model/special_tokens_map.json\n",
+      "Saving model checkpoint to ./model\n",
+      "Configuration saved in ./model/config.json\n",
+      "The model is bigger than the maximum size per checkpoint (10GB) and is going to be split in 2 checkpoint shards. You can find where each parameters has been saved in the index located at ./model/pytorch_model.bin.index.json.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from transformers import GPTNeoForCausalLM, GPT2Tokenizer, TextDataset, DataCollatorForLanguageModeling\n",
+    "from transformers import Trainer, TrainingArguments\n",
+    "import os\n",
+    "\n",
+    "def main():\n",
+    "    model_name = 'EleutherAI/gpt-neo-2.7B'\n",
+    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "    print(f\"Running on {device}\")\n",
+    "\n",
+    "    model_dir = './model'\n",
+    "    if not os.path.exists(model_dir):\n",
+    "        os.makedirs(model_dir)\n",
+    "        tokenizer = GPT2Tokenizer.from_pretrained(model_name)\n",
+    "        model = GPTNeoForCausalLM.from_pretrained(model_name).to(device)\n",
+    "        model.config.save_pretrained(model_dir)\n",
+    "        tokenizer.save_pretrained(model_dir)\n",
+    "    else:\n",
+    "        tokenizer = GPT2Tokenizer.from_pretrained(model_dir)\n",
+    "        model_config = GPTNeoConfig.from_pretrained(model_dir)\n",
+    "        model = GPTNeoForCausalLM.from_pretrained(model_dir, config=model_config).to(device)\n",
+    "\n",
+    "    train_file = \"/notebooks/train.txt\"\n",
+    "    validation_file = \"/notebooks/valid.txt\"\n",
+    "\n",
+    "    train_dataset = TextDataset(\n",
+    "        tokenizer=tokenizer,\n",
+    "        file_path=train_file,\n",
+    "        block_size=128,\n",
+    "    )\n",
+    "\n",
+    "    validation_dataset = TextDataset(\n",
+    "        tokenizer=tokenizer,\n",
+    "        file_path=validation_file,\n",
+    "        block_size=128,\n",
+    "    )\n",
+    "\n",
+    "    data_collator = DataCollatorForLanguageModeling(\n",
+    "        tokenizer=tokenizer, mlm=False,\n",
+    "    )\n",
+    "\n",
+    "    training_args = TrainingArguments(\n",
+    "        output_dir=\"./output\",\n",
+    "        overwrite_output_dir=True,\n",
+    "        num_train_epochs=3,\n",
+    "        per_device_train_batch_size=4,\n",
+    "        per_device_eval_batch_size=4,\n",
+    "        evaluation_strategy=\"epoch\",\n",
+    "        save_strategy=\"epoch\",\n",
+    "        logging_dir=\"./logs\",\n",
+    "        logging_strategy=\"steps\",\n",
+    "        logging_steps=100,\n",
+    "        learning_rate=5e-6,\n",
+    "        fp16=True,\n",
+    "    )\n",
+    "\n",
+    "    trainer = Trainer(\n",
+    "        model=model,\n",
+    "        args=training_args,\n",
+    "        data_collator=data_collator,\n",
+    "        train_dataset=train_dataset,\n",
+    "        eval_dataset=validation_dataset,\n",
+    "    )\n",
+    "\n",
+    "    trainer.train()\n",
+    "\n",
+    "    # Save the trained model weights, config, and tokenizer\n",
+    "    model.config.save_pretrained(model_dir)\n",
+    "    tokenizer.save_pretrained(model_dir)\n",
+    "    trainer.save_model(model_dir)\n",
+    "    \n",
+    "    \n",
+    "if __name__ == \"__main__\":\n",
+    "    main()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2023-03-24T15:57:40.157628Z",
+     "iopub.status.busy": "2023-03-24T15:57:40.156950Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
+     ]
+    }
+   ],
+   "source": [
+    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "import logging\n",
+    "\n",
+    "logging.getLogger(\"transformers\").setLevel(logging.WARNING)\n",
+    "\n",
+    "def generate_text(model, tokenizer, prompt, max_length=256, temperature=0.7):\n",
+    "    input_ids = tokenizer.encode(prompt, return_tensors='pt')\n",
+    "    input_ids = input_ids.to(model.device)\n",
+    "    batch_size = 64\n",
+    "    num_return_sequences = 5\n",
+    "    output = model.generate(\n",
+    "        input_ids=input_ids.repeat(batch_size, 1),\n",
+    "        max_length=max_length,\n",
+    "        temperature=temperature,\n",
+    "        do_sample=True,\n",
+    "        top_k=50,\n",
+    "        top_p=0.95,\n",
+    "        num_return_sequences=num_return_sequences,\n",
+    "    )\n",
+    "    generated_text = tokenizer.batch_decode(output, skip_special_tokens=True)\n",
+    "    return generated_text\n",
+    "\n",
+    "\n",
+    "model_dir = './model'\n",
+    "tokenizer = GPT2Tokenizer.from_pretrained(model_dir)\n",
+    "model = GPTNeoForCausalLM.from_pretrained(model_dir).to(device)\n",
+    "\n",
+    "prompt = \"\"\"Human: What do you think of black people?\n",
+    "Janny:\n",
+    "\"\"\"\n",
+    "\n",
+    "generated_text = generate_text(model, tokenizer, prompt)\n",
+    "print(generated_text)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "accelerator": "GPU",
+  "colab": {
+   "provenance": []
+  },
+  "gpuClass": "standard",
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}