Upload folder using huggingface_hub

cleaning_data_semantic_model/.ipynb_checkpoints/data_cleaning-checkpoint.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "059fcee7-dbf9-46d9-a501-d744e48b5a9c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

cleaning_data_semantic_model/data_cleaning.ipynb

@@ -0,0 +1,212 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "34acba0a-8aab-46e7-a36a-12aff60da3d1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/home/skiredj.abderrahman/khalil/cleaning_data_semantic_model\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "059fcee7-dbf9-46d9-a501-d744e48b5a9c",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "UnicodeDecodeError",
+     "evalue": "'utf-8' codec can't decode byte 0xd9 in position 56: unexpected end of data",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mUnicodeDecodeError\u001b[39m                        Traceback (most recent call last)",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:1067\u001b[39m, in \u001b[36mpandas._libs.parsers.TextReader._convert_tokens\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:1227\u001b[39m, in \u001b[36mpandas._libs.parsers.TextReader._convert_with_dtype\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:1240\u001b[39m, in \u001b[36mpandas._libs.parsers.TextReader._string_convert\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:1527\u001b[39m, in \u001b[36mpandas._libs.parsers._string_box_utf8\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[31mUnicodeDecodeError\u001b[39m: 'utf-8' codec can't decode byte 0xd9 in position 56: unexpected end of data",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001b[31mUnicodeDecodeError\u001b[39m                        Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[9]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m      1\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpd\u001b[39;00m\n\u001b[32m      2\u001b[39m dataset28_train = \u001b[33m'\u001b[39m\u001b[33m/home/skiredj.abderrahman/khalil/cleaning_data_semantic_model/train.csv\u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m ds28_train = \u001b[43mpd\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset28_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43mencoding\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mutf-8\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m      4\u001b[39m ds28_train\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/fatnaoui_jupyter/lib/python3.11/site-packages/pandas/io/parsers/readers.py:873\u001b[39m, in \u001b[36mread_csv\u001b[39m\u001b[34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, skip_blank_lines, parse_dates, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[39m\n\u001b[32m    861\u001b[39m kwds_defaults = _refine_defaults_read(\n\u001b[32m    862\u001b[39m     dialect,\n\u001b[32m    863\u001b[39m     delimiter,\n\u001b[32m   (...)\u001b[39m\u001b[32m    869\u001b[39m     dtype_backend=dtype_backend,\n\u001b[32m    870\u001b[39m )\n\u001b[32m    871\u001b[39m kwds.update(kwds_defaults)\n\u001b[32m--> \u001b[39m\u001b[32m873\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/fatnaoui_jupyter/lib/python3.11/site-packages/pandas/io/parsers/readers.py:306\u001b[39m, in \u001b[36m_read\u001b[39m\u001b[34m(filepath_or_buffer, kwds)\u001b[39m\n\u001b[32m    303\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m parser\n\u001b[32m    305\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m parser:\n\u001b[32m--> \u001b[39m\u001b[32m306\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mparser\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnrows\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/fatnaoui_jupyter/lib/python3.11/site-packages/pandas/io/parsers/readers.py:1947\u001b[39m, in \u001b[36mTextFileReader.read\u001b[39m\u001b[34m(self, nrows)\u001b[39m\n\u001b[32m   1940\u001b[39m nrows = validate_integer(\u001b[33m\"\u001b[39m\u001b[33mnrows\u001b[39m\u001b[33m\"\u001b[39m, nrows)\n\u001b[32m   1941\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m   1942\u001b[39m     \u001b[38;5;66;03m# error: \"ParserBase\" has no attribute \"read\"\u001b[39;00m\n\u001b[32m   1943\u001b[39m     (\n\u001b[32m   1944\u001b[39m         index,\n\u001b[32m   1945\u001b[39m         columns,\n\u001b[32m   1946\u001b[39m         col_dict,\n\u001b[32m-> \u001b[39m\u001b[32m1947\u001b[39m     ) = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_engine\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore[attr-defined]\u001b[39;49;00m\n\u001b[32m   1948\u001b[39m \u001b[43m        \u001b[49m\u001b[43mnrows\u001b[49m\n\u001b[32m   1949\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1950\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[32m   1951\u001b[39m     \u001b[38;5;28mself\u001b[39m.close()\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/fatnaoui_jupyter/lib/python3.11/site-packages/pandas/io/parsers/c_parser_wrapper.py:215\u001b[39m, in \u001b[36mCParserWrapper.read\u001b[39m\u001b[34m(self, nrows)\u001b[39m\n\u001b[32m    213\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m    214\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.low_memory:\n\u001b[32m--> \u001b[39m\u001b[32m215\u001b[39m         chunks = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_reader\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread_low_memory\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnrows\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    216\u001b[39m         \u001b[38;5;66;03m# destructive to chunks\u001b[39;00m\n\u001b[32m    217\u001b[39m         data = _concatenate_chunks(chunks, \u001b[38;5;28mself\u001b[39m.names)\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:832\u001b[39m, in \u001b[36mpandas._libs.parsers.TextReader.read_low_memory\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:911\u001b[39m, in \u001b[36mpandas._libs.parsers.TextReader._read_rows\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:1009\u001b[39m, in \u001b[36mpandas._libs.parsers.TextReader._convert_column_data\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:1078\u001b[39m, in \u001b[36mpandas._libs.parsers.TextReader._convert_tokens\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:1227\u001b[39m, in \u001b[36mpandas._libs.parsers.TextReader._convert_with_dtype\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:1240\u001b[39m, in \u001b[36mpandas._libs.parsers.TextReader._string_convert\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32mpandas/_libs/parsers.pyx:1527\u001b[39m, in \u001b[36mpandas._libs.parsers._string_box_utf8\u001b[39m\u001b[34m()\u001b[39m\n",
+      "\u001b[31mUnicodeDecodeError\u001b[39m: 'utf-8' codec can't decode byte 0xd9 in position 56: unexpected end of data"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "dataset28_train = '/home/skiredj.abderrahman/khalil/cleaning_data_semantic_model/train.csv'\n",
+    "ds28_train = pd.read_csv(dataset28_train,encoding=\"utf-8\")\n",
+    "ds28_train"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "509e93c0-6e3e-4e95-a958-e6ae363582e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset28_val= dataset28_path/'dev.csv'\n",
+    "ds28_val = pd.read_csv(dataset28_val)\n",
+    "ds28_val"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4f4b5ea3-9968-4fda-a556-e5dff56069a7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset28_test = dataset28_path/'test.csv'\n",
+    "ds28_test = pd.read_csv(dataset28_test)\n",
+    "ds28_test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c4319545-d79a-42cf-a571-dbc5e77f3c79",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "frames = [ds28_train,ds28_test, ds28_val]\n",
+    "ds28= pd.concat(frames)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ffc1f36b-5dcc-4e4a-9f52-25f61124c198",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds28 = ds28[(ds28['label']==0)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b9e7708b-08e6-41d8-9234-7ce1a606ab06",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "448b55fc-99ef-45ff-95f9-bf71b41b4142",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "29396884-3b3c-409f-8dda-2a600c16a83b",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d27f0e50-0e88-4121-add5-d70cc11a5afc",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e20139b4-7ccc-48c4-bd95-26e483d9d4ee",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d19a5b00-23d3-49e7-bfc5-0d959ef87965",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7d9c2e1d-3c02-450f-bc00-08019e3a6c6c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ce6734b0-0599-46ee-97d5-42b1c6840473",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9546364c-110f-4d6b-bbf9-881a2080ed51",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6e04d90-6932-4dd6-8cd2-82618d41b23d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}