Upload prep.ipynb

prep.ipynb

@@ -0,0 +1,248 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3282ec4a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_auth.py:138: UserWarning: \n",
+      "Error while fetching `HF_TOKEN` secret value from your vault: 'Requesting secret HF_TOKEN timed out. Secrets can only be fetched when running from the Colab UI.'.\n",
+      "  warnings.warn(f\"\\nError while fetching `HF_TOKEN` secret value from your vault: '{str(e)}'.\")\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "07435d1af4f64845b0d67576071b74bc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "README.md:   0%|          | 0.00/26.4k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
+      "WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e0550b531d5840b9b7c57f98084d036a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Resolving data files:   0%|          | 0/2410 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "dfba8eef1d774165854440c0bcbad299",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Resolving data files:   0%|          | 0/2410 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "import numpy as np\n",
+    "import tiktoken\n",
+    "\n",
+    "enc = tiktoken.get_encoding(\"gpt2\")\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "dataset = load_dataset(\n",
+    "    \"HuggingFaceFW/fineweb-edu\",\n",
+    "    split=\"train\",\n",
+    "    streaming=True\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "bb9ff2be",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9b5d482ebadf4b388c643ec03b11be00",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading bytes:           |  0.00B            "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9a9ec6fcc7df4b26814f151544b62dc6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Reconstructing (incomplete total...): |          |  0.00B /  0.00B            "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "bc7bb2435f2a4f78b93f64e15c2b1465",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Fetching 5 files:   0%|          | 0/5 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.google.colaboratory.intrinsic+json": {
+       "type": "string"
+      },
+      "text/plain": [
+       "'/content/tokensstore'"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import warnings\n",
+    "from huggingface_hub import HfApi\n",
+    "warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
+    "warnings.filterwarnings(\"ignore\", category=FutureWarning)\n",
+    "\n",
+    "from huggingface_hub import snapshot_download\n",
+    "from huggingface_hub import hf_hub_download\n",
+    "\n",
+    "snapshot_download(\n",
+    "    repo_id=\"MLGOATKOJI/preprocceses\",\n",
+    "    repo_type=\"dataset\",\n",
+    "    local_dir=\"./tokensstore\"\n",
+    ")\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "53fa23c2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "enc = tiktoken.get_encoding(\"gpt2\")\n",
+    "target_tokens = 5_000_000_000\n",
+    "total = 0\n",
+    "\n",
+    "with open(\"./tokensstore/5btokens.bin\", \"wb\") as out:\n",
+    "\n",
+    "    for item in dataset:\n",
+    "        \n",
+    "        if total >= target_tokens:\n",
+    "            break\n",
+    "\n",
+    "        ids = enc.encode(\n",
+    "            item[\"text\"]\n",
+    "        )\n",
+    "\n",
+    "        remaining = target_tokens - total\n",
+    "\n",
+    "        # Trim if this document exceeds remaining tokens\n",
+    "        if len(ids) > remaining:\n",
+    "            ids = ids[:remaining]\n",
+    "\n",
+    "        ids = np.array(ids, dtype=np.uint16)\n",
+    "        ids.tofile(out)\n",
+    "\n",
+    "        total += len(ids)\n",
+    "\n",
+    "        print(f\"{total:,}/{target_tokens}\")#why isnt this ending anything to my train.bin?"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e68f049c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.google.colaboratory.intrinsic+json": {
+       "type": "string"
+      },
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/datasets/MLGOATKOJI/preprocceses/commit/15019f40f6b650d77448fdc7afa2e7c028288e5d', commit_message='Upload folder using huggingface_hub', commit_description='', oid='15019f40f6b650d77448fdc7afa2e7c028288e5d', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/MLGOATKOJI/preprocceses', endpoint='https://huggingface.co', repo_type='dataset', repo_id='MLGOATKOJI/preprocceses'), pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\n",
+    "from huggingface_hub import login\n",
+    "from huggingface_hub import HfApi\n",
+    "\n",
+    "login()\n",
+    "\n",
+    "api.upload_folder(\n",
+    "    folder_path=\"./tokensstore\",\n",
+    "    repo_id=\"MLGOATKOJI/preprocceses\",\n",
+    "    repo_type=\"dataset\"\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}