Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- bart/data/Untitled.ipynb +778 -0
- bart/data/tokenizer/tokenizer.json +155 -0
- bart/data/tokenizer/tokenizer_config.json +13 -0
- bart/data/tokenizer/vocab.txt +0 -0
- data/data-bin/dict.eng_Latn.txt +0 -0
- data/data-bin/dict.mni_Beng.txt +0 -0
- data/data-bin/preprocess.log +14 -0
- data/data-bin/test.mni_Beng-eng_Latn.eng_Latn.idx +0 -0
- data/data-bin/test.mni_Beng-eng_Latn.mni_Beng.idx +0 -0
- data/data-bin/valid.mni_Beng-eng_Latn.eng_Latn.idx +0 -0
- data/data-bin/valid.mni_Beng-eng_Latn.mni_Beng.idx +0 -0
- data/dev.eng_Latn +0 -0
- data/dev.mni_Beng +0 -0
- data/fairseq/.gitignore +141 -0
- data/fairseq/.gitmodules +4 -0
- data/fairseq/.pre-commit-config.yaml +40 -0
- data/fairseq/CODE_OF_CONDUCT.md +77 -0
- data/fairseq/CONTRIBUTING.md +82 -0
- data/fairseq/LICENSE +21 -0
- data/fairseq/MANIFEST.in +1 -0
- data/fairseq/README.md +242 -0
- data/fairseq/RELEASE.md +13 -0
- data/fairseq/fairseq/__pycache__/__init__.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/binarizer.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/checkpoint_utils.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/file_chunker_utils.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/file_io.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/file_utils.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/hub_utils.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/incremental_decoding_utils.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/iterative_refinement_generator.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/nan_detector.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/ngram_repeat_block.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/options.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/pdb.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/quantization_utils.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/registry.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/search.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/sequence_generator.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/speech_generator.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/token_generation_constraints.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/tokenizer.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/trainer.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/utils.cpython-38.pyc +0 -0
- data/fairseq/fairseq/__pycache__/version.cpython-38.pyc +0 -0
- data/fairseq/fairseq/dataclass/__init__.py +13 -0
- data/fairseq/fairseq/dataclass/__pycache__/__init__.cpython-38.pyc +0 -0
- data/fairseq/fairseq/dataclass/__pycache__/configs.cpython-38.pyc +0 -0
- data/fairseq/fairseq/dataclass/__pycache__/constants.cpython-38.pyc +0 -0
- data/fairseq/fairseq/dataclass/__pycache__/initialize.cpython-38.pyc +0 -0
bart/data/Untitled.ipynb
ADDED
|
@@ -0,0 +1,778 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "code",
|
| 5 |
+
"execution_count": 9,
|
| 6 |
+
"id": "52fc5140-a230-4e0b-9781-11ffeeb98e8c",
|
| 7 |
+
"metadata": {},
|
| 8 |
+
"outputs": [
|
| 9 |
+
{
|
| 10 |
+
"data": {
|
| 11 |
+
"text/plain": [
|
| 12 |
+
"'/teamspace/studios/this_studio/bart'"
|
| 13 |
+
]
|
| 14 |
+
},
|
| 15 |
+
"execution_count": 9,
|
| 16 |
+
"metadata": {},
|
| 17 |
+
"output_type": "execute_result"
|
| 18 |
+
}
|
| 19 |
+
],
|
| 20 |
+
"source": [
|
| 21 |
+
"pwd"
|
| 22 |
+
]
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"cell_type": "code",
|
| 26 |
+
"execution_count": 7,
|
| 27 |
+
"id": "aa457523-db88-4268-86f5-3e7a16c6a8e4",
|
| 28 |
+
"metadata": {},
|
| 29 |
+
"outputs": [
|
| 30 |
+
{
|
| 31 |
+
"name": "stdout",
|
| 32 |
+
"output_type": "stream",
|
| 33 |
+
"text": [
|
| 34 |
+
"/teamspace/studios/this_studio/bart\n"
|
| 35 |
+
]
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "stderr",
|
| 39 |
+
"output_type": "stream",
|
| 40 |
+
"text": [
|
| 41 |
+
"/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/IPython/core/magics/osm.py:417: UserWarning: using dhist requires you to install the `pickleshare` library.\n",
|
| 42 |
+
" self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
|
| 43 |
+
]
|
| 44 |
+
}
|
| 45 |
+
],
|
| 46 |
+
"source": [
|
| 47 |
+
"cd .."
|
| 48 |
+
]
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"cell_type": "code",
|
| 52 |
+
"execution_count": 13,
|
| 53 |
+
"id": "a38f801f-929a-43b4-8bf7-70ce28ff239b",
|
| 54 |
+
"metadata": {},
|
| 55 |
+
"outputs": [
|
| 56 |
+
{
|
| 57 |
+
"name": "stdout",
|
| 58 |
+
"output_type": "stream",
|
| 59 |
+
"text": [
|
| 60 |
+
"/teamspace/studios/this_studio/bart/data\n"
|
| 61 |
+
]
|
| 62 |
+
}
|
| 63 |
+
],
|
| 64 |
+
"source": [
|
| 65 |
+
"cd data"
|
| 66 |
+
]
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"cell_type": "code",
|
| 70 |
+
"execution_count": 11,
|
| 71 |
+
"id": "f31bec58-2356-4a9e-bc49-698de2f8ab18",
|
| 72 |
+
"metadata": {},
|
| 73 |
+
"outputs": [
|
| 74 |
+
{
|
| 75 |
+
"name": "stdout",
|
| 76 |
+
"output_type": "stream",
|
| 77 |
+
"text": [
|
| 78 |
+
"/teamspace/studios/this_studio/bart\n"
|
| 79 |
+
]
|
| 80 |
+
}
|
| 81 |
+
],
|
| 82 |
+
"source": [
|
| 83 |
+
"import os\n",
|
| 84 |
+
"print(os.getcwd())"
|
| 85 |
+
]
|
| 86 |
+
},
|
| 87 |
+
{
|
| 88 |
+
"cell_type": "code",
|
| 89 |
+
"execution_count": 16,
|
| 90 |
+
"id": "82d4110f-88a9-4386-bea6-0176f9bf1053",
|
| 91 |
+
"metadata": {},
|
| 92 |
+
"outputs": [
|
| 93 |
+
{
|
| 94 |
+
"name": "stdout",
|
| 95 |
+
"output_type": "stream",
|
| 96 |
+
"text": [
|
| 97 |
+
"True\n"
|
| 98 |
+
]
|
| 99 |
+
}
|
| 100 |
+
],
|
| 101 |
+
"source": [
|
| 102 |
+
"import os\n",
|
| 103 |
+
"print(os.path.exists(\"mni_train_tokenise.txt\"))"
|
| 104 |
+
]
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"cell_type": "code",
|
| 108 |
+
"execution_count": 1,
|
| 109 |
+
"id": "b1600542-f812-498b-85dc-9c6c68eb71f1",
|
| 110 |
+
"metadata": {},
|
| 111 |
+
"outputs": [],
|
| 112 |
+
"source": [
|
| 113 |
+
"import os\n",
|
| 114 |
+
"os.makedirs(\"./tokenizer\", exist_ok=True)"
|
| 115 |
+
]
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"cell_type": "code",
|
| 119 |
+
"execution_count": 4,
|
| 120 |
+
"id": "1df58de9-17fa-4ddb-9f9d-37f3883cd475",
|
| 121 |
+
"metadata": {},
|
| 122 |
+
"outputs": [
|
| 123 |
+
{
|
| 124 |
+
"data": {
|
| 125 |
+
"text/plain": [
|
| 126 |
+
"['./tokenizer/vocab.txt']"
|
| 127 |
+
]
|
| 128 |
+
},
|
| 129 |
+
"execution_count": 4,
|
| 130 |
+
"metadata": {},
|
| 131 |
+
"output_type": "execute_result"
|
| 132 |
+
}
|
| 133 |
+
],
|
| 134 |
+
"source": [
|
| 135 |
+
"tokenizer.save_model(\"./tokenizer\")"
|
| 136 |
+
]
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"cell_type": "code",
|
| 140 |
+
"execution_count": 7,
|
| 141 |
+
"id": "1e4a3381-d633-4900-9665-fe485d0aaf2d",
|
| 142 |
+
"metadata": {},
|
| 143 |
+
"outputs": [
|
| 144 |
+
{
|
| 145 |
+
"name": "stdout",
|
| 146 |
+
"output_type": "stream",
|
| 147 |
+
"text": [
|
| 148 |
+
"\n",
|
| 149 |
+
"\n",
|
| 150 |
+
"\n"
|
| 151 |
+
]
|
| 152 |
+
},
|
| 153 |
+
{
|
| 154 |
+
"data": {
|
| 155 |
+
"text/plain": [
|
| 156 |
+
"['./tokenizer/vocab.txt']"
|
| 157 |
+
]
|
| 158 |
+
},
|
| 159 |
+
"execution_count": 7,
|
| 160 |
+
"metadata": {},
|
| 161 |
+
"output_type": "execute_result"
|
| 162 |
+
}
|
| 163 |
+
],
|
| 164 |
+
"source": [
|
| 165 |
+
"from tokenizers import BertWordPieceTokenizer\n",
|
| 166 |
+
"import os\n",
|
| 167 |
+
"\n",
|
| 168 |
+
"# Train tokenizer\n",
|
| 169 |
+
"tokenizer = BertWordPieceTokenizer(lowercase=True)\n",
|
| 170 |
+
"\n",
|
| 171 |
+
"tokenizer.train(\n",
|
| 172 |
+
" files=[\"mni_train_tokenise.txt\"],\n",
|
| 173 |
+
" vocab_size=30000,\n",
|
| 174 |
+
" min_frequency=2,\n",
|
| 175 |
+
" special_tokens=[\"[PAD]\", \"[UNK]\", \"[CLS]\", \"[SEP]\", \"[MASK]\"]\n",
|
| 176 |
+
")\n",
|
| 177 |
+
"\n",
|
| 178 |
+
"# ✅ Create directory first\n",
|
| 179 |
+
"os.makedirs(\"./tokenizer\", exist_ok=True)\n",
|
| 180 |
+
"\n",
|
| 181 |
+
"# Save tokenizer\n",
|
| 182 |
+
"tokenizer.save_model(\"./tokenizer\")"
|
| 183 |
+
]
|
| 184 |
+
},
|
| 185 |
+
{
|
| 186 |
+
"cell_type": "code",
|
| 187 |
+
"execution_count": 8,
|
| 188 |
+
"id": "f429d192-94b3-492f-8d53-e6e609e44a29",
|
| 189 |
+
"metadata": {},
|
| 190 |
+
"outputs": [
|
| 191 |
+
{
|
| 192 |
+
"data": {
|
| 193 |
+
"text/plain": [
|
| 194 |
+
"('./tokenizer/tokenizer_config.json', './tokenizer/tokenizer.json')"
|
| 195 |
+
]
|
| 196 |
+
},
|
| 197 |
+
"execution_count": 8,
|
| 198 |
+
"metadata": {},
|
| 199 |
+
"output_type": "execute_result"
|
| 200 |
+
}
|
| 201 |
+
],
|
| 202 |
+
"source": [
|
| 203 |
+
"from transformers import BertTokenizerFast\n",
|
| 204 |
+
"\n",
|
| 205 |
+
"tokenizer = BertTokenizerFast(\n",
|
| 206 |
+
" vocab_file=\"./tokenizer/vocab.txt\",\n",
|
| 207 |
+
" do_lower_case=True\n",
|
| 208 |
+
")\n",
|
| 209 |
+
"\n",
|
| 210 |
+
"# Optional (recommended)\n",
|
| 211 |
+
"tokenizer.save_pretrained(\"./tokenizer\")"
|
| 212 |
+
]
|
| 213 |
+
},
|
| 214 |
+
{
|
| 215 |
+
"cell_type": "code",
|
| 216 |
+
"execution_count": 9,
|
| 217 |
+
"id": "859a1114-dc93-439f-8360-1d9212cc7392",
|
| 218 |
+
"metadata": {},
|
| 219 |
+
"outputs": [],
|
| 220 |
+
"source": [
|
| 221 |
+
"# 4. Load Dataset\n",
|
| 222 |
+
"# =========================\n",
|
| 223 |
+
"from datasets import load_dataset\n",
|
| 224 |
+
"\n",
|
| 225 |
+
"dataset = load_dataset(\"text\", data_files={\"train\": \"mni_train_tokenise.txt\"})\n",
|
| 226 |
+
"\n",
|
| 227 |
+
"# =========================\n",
|
| 228 |
+
"# 5. Tokenization (NO padding, NO truncation)\n",
|
| 229 |
+
"# =========================\n",
|
| 230 |
+
"def tokenize_function(examples):\n",
|
| 231 |
+
" return tokenizer(examples[\"text\"]) # important change\n",
|
| 232 |
+
"\n",
|
| 233 |
+
"tokenized_dataset = dataset.map(\n",
|
| 234 |
+
" tokenize_function,\n",
|
| 235 |
+
" batched=True,\n",
|
| 236 |
+
" remove_columns=[\"text\"]\n",
|
| 237 |
+
")"
|
| 238 |
+
]
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"cell_type": "code",
|
| 242 |
+
"execution_count": 7,
|
| 243 |
+
"id": "7224b2c1-4908-44ba-92f4-e87583412ea3",
|
| 244 |
+
"metadata": {},
|
| 245 |
+
"outputs": [],
|
| 246 |
+
"source": [
|
| 247 |
+
"# 6. Group Texts into 128 chunks\n",
|
| 248 |
+
"# =========================\n",
|
| 249 |
+
"def group_texts(examples):\n",
|
| 250 |
+
" concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n",
|
| 251 |
+
" total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
|
| 252 |
+
"\n",
|
| 253 |
+
" # drop remainder\n",
|
| 254 |
+
" total_length = (total_length // 128) * 128\n",
|
| 255 |
+
"\n",
|
| 256 |
+
" result = {\n",
|
| 257 |
+
" k: [t[i:i + 128] for i in range(0, total_length, 128)]\n",
|
| 258 |
+
" for k, t in concatenated_examples.items()\n",
|
| 259 |
+
" }\n",
|
| 260 |
+
" return result\n",
|
| 261 |
+
"\n",
|
| 262 |
+
"lm_dataset = tokenized_dataset.map(group_texts, batched=True)"
|
| 263 |
+
]
|
| 264 |
+
},
|
| 265 |
+
{
|
| 266 |
+
"cell_type": "code",
|
| 267 |
+
"execution_count": 10,
|
| 268 |
+
"id": "36a9d0d0-4442-411b-abea-f8b53e6354d5",
|
| 269 |
+
"metadata": {},
|
| 270 |
+
"outputs": [
|
| 271 |
+
{
|
| 272 |
+
"name": "stderr",
|
| 273 |
+
"output_type": "stream",
|
| 274 |
+
"text": [
|
| 275 |
+
"/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/transformers/data/data_collator.py:1028: FutureWarning: DataCollatorForWholeWordMask is deprecated and will be removed in a future version, you can now use DataCollatorForLanguageModeling with whole_word_mask=True instead.\n",
|
| 276 |
+
" warnings.warn(\n"
|
| 277 |
+
]
|
| 278 |
+
}
|
| 279 |
+
],
|
| 280 |
+
"source": [
|
| 281 |
+
"# =========================\n",
|
| 282 |
+
"# 7. Whole Word Masking\n",
|
| 283 |
+
"# =========================\n",
|
| 284 |
+
"from transformers import DataCollatorForWholeWordMask\n",
|
| 285 |
+
"\n",
|
| 286 |
+
"data_collator = DataCollatorForWholeWordMask(\n",
|
| 287 |
+
" tokenizer=tokenizer,\n",
|
| 288 |
+
" mlm=True,\n",
|
| 289 |
+
" mlm_probability=0.15\n",
|
| 290 |
+
")\n"
|
| 291 |
+
]
|
| 292 |
+
},
|
| 293 |
+
{
|
| 294 |
+
"cell_type": "code",
|
| 295 |
+
"execution_count": 13,
|
| 296 |
+
"id": "e761e52b-a953-4b0b-a3f4-328bbf049fa7",
|
| 297 |
+
"metadata": {},
|
| 298 |
+
"outputs": [
|
| 299 |
+
{
|
| 300 |
+
"name": "stdout",
|
| 301 |
+
"output_type": "stream",
|
| 302 |
+
"text": [
|
| 303 |
+
"Collecting accelerate\n",
|
| 304 |
+
" Downloading accelerate-1.13.0-py3-none-any.whl.metadata (19 kB)\n",
|
| 305 |
+
"Requirement already satisfied: numpy>=1.17 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from accelerate) (1.26.4)\n",
|
| 306 |
+
"Requirement already satisfied: packaging>=20.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from accelerate) (25.0)\n",
|
| 307 |
+
"Requirement already satisfied: psutil in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from accelerate) (7.2.2)\n",
|
| 308 |
+
"Requirement already satisfied: pyyaml in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from accelerate) (6.0.3)\n",
|
| 309 |
+
"Requirement already satisfied: torch>=2.0.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from accelerate) (2.8.0+cu128)\n",
|
| 310 |
+
"Requirement already satisfied: huggingface_hub>=0.21.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from accelerate) (1.9.0)\n",
|
| 311 |
+
"Requirement already satisfied: safetensors>=0.4.3 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from accelerate) (0.7.0)\n",
|
| 312 |
+
"Requirement already satisfied: filelock>=3.10.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from huggingface_hub>=0.21.0->accelerate) (3.25.2)\n",
|
| 313 |
+
"Requirement already satisfied: fsspec>=2023.5.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from huggingface_hub>=0.21.0->accelerate) (2026.2.0)\n",
|
| 314 |
+
"Requirement already satisfied: hf-xet<2.0.0,>=1.4.3 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from huggingface_hub>=0.21.0->accelerate) (1.4.3)\n",
|
| 315 |
+
"Requirement already satisfied: httpx<1,>=0.23.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from huggingface_hub>=0.21.0->accelerate) (0.28.1)\n",
|
| 316 |
+
"Requirement already satisfied: tqdm>=4.42.1 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from huggingface_hub>=0.21.0->accelerate) (4.67.3)\n",
|
| 317 |
+
"Requirement already satisfied: typer in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from huggingface_hub>=0.21.0->accelerate) (0.24.1)\n",
|
| 318 |
+
"Requirement already satisfied: typing-extensions>=4.1.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from huggingface_hub>=0.21.0->accelerate) (4.15.0)\n",
|
| 319 |
+
"Requirement already satisfied: anyio in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (4.12.1)\n",
|
| 320 |
+
"Requirement already satisfied: certifi in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (2026.2.25)\n",
|
| 321 |
+
"Requirement already satisfied: httpcore==1.* in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (1.0.9)\n",
|
| 322 |
+
"Requirement already satisfied: idna in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (3.11)\n",
|
| 323 |
+
"Requirement already satisfied: h11>=0.16 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->huggingface_hub>=0.21.0->accelerate) (0.16.0)\n",
|
| 324 |
+
"Requirement already satisfied: setuptools in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (80.10.2)\n",
|
| 325 |
+
"Requirement already satisfied: sympy>=1.13.3 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (1.14.0)\n",
|
| 326 |
+
"Requirement already satisfied: networkx in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (3.6.1)\n",
|
| 327 |
+
"Requirement already satisfied: jinja2 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (3.1.6)\n",
|
| 328 |
+
"Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (12.8.93)\n",
|
| 329 |
+
"Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (12.8.90)\n",
|
| 330 |
+
"Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (12.8.90)\n",
|
| 331 |
+
"Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (9.10.2.21)\n",
|
| 332 |
+
"Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (12.8.4.1)\n",
|
| 333 |
+
"Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (11.3.3.83)\n",
|
| 334 |
+
"Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (10.3.9.90)\n",
|
| 335 |
+
"Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (11.7.3.90)\n",
|
| 336 |
+
"Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (12.5.8.93)\n",
|
| 337 |
+
"Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (0.7.1)\n",
|
| 338 |
+
"Requirement already satisfied: nvidia-nccl-cu12==2.27.3 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (2.27.3)\n",
|
| 339 |
+
"Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (12.8.90)\n",
|
| 340 |
+
"Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (12.8.93)\n",
|
| 341 |
+
"Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (1.13.1.3)\n",
|
| 342 |
+
"Requirement already satisfied: triton==3.4.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from torch>=2.0.0->accelerate) (3.4.0)\n",
|
| 343 |
+
"Requirement already satisfied: mpmath<1.4,>=1.1.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from sympy>=1.13.3->torch>=2.0.0->accelerate) (1.3.0)\n",
|
| 344 |
+
"Requirement already satisfied: MarkupSafe>=2.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from jinja2->torch>=2.0.0->accelerate) (3.0.3)\n",
|
| 345 |
+
"Requirement already satisfied: click>=8.2.1 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from typer->huggingface_hub>=0.21.0->accelerate) (8.3.1)\n",
|
| 346 |
+
"Requirement already satisfied: shellingham>=1.3.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from typer->huggingface_hub>=0.21.0->accelerate) (1.5.4)\n",
|
| 347 |
+
"Requirement already satisfied: rich>=12.3.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from typer->huggingface_hub>=0.21.0->accelerate) (14.3.3)\n",
|
| 348 |
+
"Requirement already satisfied: annotated-doc>=0.0.2 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from typer->huggingface_hub>=0.21.0->accelerate) (0.0.4)\n",
|
| 349 |
+
"Requirement already satisfied: markdown-it-py>=2.2.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from rich>=12.3.0->typer->huggingface_hub>=0.21.0->accelerate) (4.0.0)\n",
|
| 350 |
+
"Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from rich>=12.3.0->typer->huggingface_hub>=0.21.0->accelerate) (2.19.2)\n",
|
| 351 |
+
"Requirement already satisfied: mdurl~=0.1 in /home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich>=12.3.0->typer->huggingface_hub>=0.21.0->accelerate) (0.1.2)\n",
|
| 352 |
+
"Downloading accelerate-1.13.0-py3-none-any.whl (383 kB)\n",
|
| 353 |
+
"Installing collected packages: accelerate\n",
|
| 354 |
+
"Successfully installed accelerate-1.13.0\n",
|
| 355 |
+
"Note: you may need to restart the kernel to use updated packages.\n"
|
| 356 |
+
]
|
| 357 |
+
}
|
| 358 |
+
],
|
| 359 |
+
"source": [
|
| 360 |
+
"pip install -U accelerate"
|
| 361 |
+
]
|
| 362 |
+
},
|
| 363 |
+
{
|
| 364 |
+
"cell_type": "code",
|
| 365 |
+
"execution_count": 11,
|
| 366 |
+
"id": "06257958-0267-4267-a77d-78167e52474e",
|
| 367 |
+
"metadata": {},
|
| 368 |
+
"outputs": [],
|
| 369 |
+
"source": [
|
| 370 |
+
"# =========================\n",
|
| 371 |
+
"# 8. Define Model\n",
|
| 372 |
+
"# =========================\n",
|
| 373 |
+
"from transformers import BertConfig, BertForMaskedLM\n",
|
| 374 |
+
"\n",
|
| 375 |
+
"config = BertConfig(\n",
|
| 376 |
+
" vocab_size=tokenizer.vocab_size,\n",
|
| 377 |
+
" hidden_size=768,\n",
|
| 378 |
+
" num_hidden_layers=12,\n",
|
| 379 |
+
" num_attention_heads=12,\n",
|
| 380 |
+
" intermediate_size=3072,\n",
|
| 381 |
+
" max_position_embeddings=512\n",
|
| 382 |
+
")\n",
|
| 383 |
+
"\n",
|
| 384 |
+
"model = BertForMaskedLM(config)"
|
| 385 |
+
]
|
| 386 |
+
},
|
| 387 |
+
{
|
| 388 |
+
"cell_type": "code",
|
| 389 |
+
"execution_count": 12,
|
| 390 |
+
"id": "01b1a374-9b2f-49cf-af4b-f1786934c8ce",
|
| 391 |
+
"metadata": {},
|
| 392 |
+
"outputs": [],
|
| 393 |
+
"source": [
|
| 394 |
+
"from transformers import TrainingArguments\n",
|
| 395 |
+
"\n",
|
| 396 |
+
"training_args = TrainingArguments(\n",
|
| 397 |
+
" output_dir=\"./bert-wwm\",\n",
|
| 398 |
+
" per_device_train_batch_size=8,\n",
|
| 399 |
+
" num_train_epochs=1,\n",
|
| 400 |
+
" learning_rate=5e-5,\n",
|
| 401 |
+
" save_steps=5000,\n",
|
| 402 |
+
" save_total_limit=2,\n",
|
| 403 |
+
" logging_steps=100,\n",
|
| 404 |
+
" fp16=True\n",
|
| 405 |
+
")"
|
| 406 |
+
]
|
| 407 |
+
},
|
| 408 |
+
{
|
| 409 |
+
"cell_type": "code",
|
| 410 |
+
"execution_count": 14,
|
| 411 |
+
"id": "0da255e4-c80f-4248-9e50-1b8745767166",
|
| 412 |
+
"metadata": {},
|
| 413 |
+
"outputs": [],
|
| 414 |
+
"source": [
|
| 415 |
+
"from transformers import BertConfig, BertForMaskedLM\n",
|
| 416 |
+
"\n",
|
| 417 |
+
"config = BertConfig(\n",
|
| 418 |
+
" vocab_size=tokenizer.vocab_size,\n",
|
| 419 |
+
" hidden_size=768,\n",
|
| 420 |
+
" num_hidden_layers=12,\n",
|
| 421 |
+
" num_attention_heads=12,\n",
|
| 422 |
+
" intermediate_size=3072,\n",
|
| 423 |
+
" max_position_embeddings=512\n",
|
| 424 |
+
")\n",
|
| 425 |
+
"\n",
|
| 426 |
+
"model = BertForMaskedLM(config)"
|
| 427 |
+
]
|
| 428 |
+
},
|
| 429 |
+
{
|
| 430 |
+
"cell_type": "code",
|
| 431 |
+
"execution_count": 16,
|
| 432 |
+
"id": "098e9a47-672e-4365-bf18-f0de1ec22300",
|
| 433 |
+
"metadata": {},
|
| 434 |
+
"outputs": [],
|
| 435 |
+
"source": [
|
| 436 |
+
"def tokenize_function(examples):\n",
|
| 437 |
+
" return tokenizer(examples[\"text\"])\n",
|
| 438 |
+
"\n",
|
| 439 |
+
"tokenized_dataset = dataset.map(\n",
|
| 440 |
+
" tokenize_function,\n",
|
| 441 |
+
" batched=True,\n",
|
| 442 |
+
" remove_columns=[\"text\"]\n",
|
| 443 |
+
")"
|
| 444 |
+
]
|
| 445 |
+
},
|
| 446 |
+
{
|
| 447 |
+
"cell_type": "code",
|
| 448 |
+
"execution_count": 17,
|
| 449 |
+
"id": "d28b39d6-56b0-4820-b36a-50b1b3f69dfd",
|
| 450 |
+
"metadata": {},
|
| 451 |
+
"outputs": [
|
| 452 |
+
{
|
| 453 |
+
"data": {
|
| 454 |
+
"application/vnd.jupyter.widget-view+json": {
|
| 455 |
+
"model_id": "9803e4c165204f7fb11f6cb507872382",
|
| 456 |
+
"version_major": 2,
|
| 457 |
+
"version_minor": 0
|
| 458 |
+
},
|
| 459 |
+
"text/plain": [
|
| 460 |
+
"Map: 0%| | 0/1897947 [00:00<?, ? examples/s]"
|
| 461 |
+
]
|
| 462 |
+
},
|
| 463 |
+
"metadata": {},
|
| 464 |
+
"output_type": "display_data"
|
| 465 |
+
}
|
| 466 |
+
],
|
| 467 |
+
"source": [
|
| 468 |
+
"def group_texts(examples):\n",
|
| 469 |
+
" concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n",
|
| 470 |
+
" total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
|
| 471 |
+
"\n",
|
| 472 |
+
" total_length = (total_length // 128) * 128\n",
|
| 473 |
+
"\n",
|
| 474 |
+
" result = {\n",
|
| 475 |
+
" k: [t[i:i + 128] for i in range(0, total_length, 128)]\n",
|
| 476 |
+
" for k, t in concatenated_examples.items()\n",
|
| 477 |
+
" }\n",
|
| 478 |
+
" return result\n",
|
| 479 |
+
"\n",
|
| 480 |
+
"lm_dataset = tokenized_dataset.map(group_texts, batched=True)"
|
| 481 |
+
]
|
| 482 |
+
},
|
| 483 |
+
{
|
| 484 |
+
"cell_type": "code",
|
| 485 |
+
"execution_count": 18,
|
| 486 |
+
"id": "5978b5f5-5b2a-4ee1-b826-58b75893720c",
|
| 487 |
+
"metadata": {},
|
| 488 |
+
"outputs": [],
|
| 489 |
+
"source": [
|
| 490 |
+
"# 10. Trainer\n",
|
| 491 |
+
"# =========================\n",
|
| 492 |
+
"from transformers import Trainer\n",
|
| 493 |
+
"\n",
|
| 494 |
+
"trainer = Trainer(\n",
|
| 495 |
+
" model=model,\n",
|
| 496 |
+
" args=training_args,\n",
|
| 497 |
+
" train_dataset=lm_dataset[\"train\"], # IMPORTANT CHANGE\n",
|
| 498 |
+
" data_collator=data_collator\n",
|
| 499 |
+
")"
|
| 500 |
+
]
|
| 501 |
+
},
|
| 502 |
+
{
|
| 503 |
+
"cell_type": "code",
|
| 504 |
+
"execution_count": 20,
|
| 505 |
+
"id": "62387df7-0fc2-42ce-9a86-5d725d52b491",
|
| 506 |
+
"metadata": {},
|
| 507 |
+
"outputs": [
|
| 508 |
+
{
|
| 509 |
+
"name": "stdout",
|
| 510 |
+
"output_type": "stream",
|
| 511 |
+
"text": [
|
| 512 |
+
"DatasetDict({\n",
|
| 513 |
+
" train: Dataset({\n",
|
| 514 |
+
" features: ['input_ids', 'token_type_ids', 'attention_mask'],\n",
|
| 515 |
+
" num_rows: 308200\n",
|
| 516 |
+
" })\n",
|
| 517 |
+
"})\n",
|
| 518 |
+
"{'input_ids': [2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 2, 1, 1, 1], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n"
|
| 519 |
+
]
|
| 520 |
+
}
|
| 521 |
+
],
|
| 522 |
+
"source": [
|
| 523 |
+
"print(lm_dataset)\n",
|
| 524 |
+
"print(lm_dataset[\"train\"][0])"
|
| 525 |
+
]
|
| 526 |
+
},
|
| 527 |
+
{
|
| 528 |
+
"cell_type": "code",
|
| 529 |
+
"execution_count": 21,
|
| 530 |
+
"id": "a2c18183-5896-4afa-b257-445aeaa70418",
|
| 531 |
+
"metadata": {},
|
| 532 |
+
"outputs": [
|
| 533 |
+
{
|
| 534 |
+
"name": "stdout",
|
| 535 |
+
"output_type": "stream",
|
| 536 |
+
"text": [
|
| 537 |
+
"চীফ মিনিষ্টরনা মখা তারদুনা পোপী পাম্বী থাবগী মহুৎ অতোপ্পা হিংনবগী লম্বী পীনবা লৈঙাক্না অঙম্বা মতেং পীগনি হায়বা ৱাফম থম্লদুনা মখোয় মখোয়গী খুনশিংদা করম্বা মহৈ মরোং থাবা চুনগদগে হায়বা মরি লৈনবা ওফিসিয়েলশিংগা ইনদুনা ৱারী শানবসু চীফ মিনিষ্টরনা খঙহনখি ।\n",
|
| 538 |
+
"\n",
|
| 539 |
+
"চহী খূদিংগী চাদা 33দি নুপী ওইগদবনি হায়বা অদুদগী তশেংনা থবক য়াওরিবা নুপী মশিংনা য়াম্না হেল্লি ।\n",
|
| 540 |
+
"\n",
|
| 541 |
+
"অইবা অসিনা তাইবং থাদোকখিবা হায়বসি মণিপুরী সাহি্যতগী মায়কৈদগী অচৌবা মাঙজবা অমনি ।\n",
|
| 542 |
+
"\n",
|
| 543 |
+
"মালেমগী মীয়ামনা মালেমগী শান্তি পুবা শান্নবা মখল কয়াগী ফম্বাক ওইরবসু কা হেনবা থবক তৌবশিংনা হেক হেক মালেমগী শান্নবগী ফম্বাক অসিগী চৎন - কাঙলোনশিং থুগায়নবসু হোৎনদবা নত্তে ।\n",
|
| 544 |
+
"\n",
|
| 545 |
+
"মল্টিলেটরেলিজমদা অনৌবা মায়ওন অমা পীনবা ইন্টরনেসনেল কম্ম্যুনিটিদা প্রেসর পীরদুনা পিএম মোদিনা হায় , মোদর্ন টেক্নোলোজিনা পুন্সিগী হীরম খুদিংমত্তা অহোংবা পুরক্লি ।\n",
|
| 546 |
+
"\n"
|
| 547 |
+
]
|
| 548 |
+
}
|
| 549 |
+
],
|
| 550 |
+
"source": [
|
| 551 |
+
"with open(\"mni_train_tokenise.txt\", encoding=\"utf-8\") as f:\n",
|
| 552 |
+
" for _ in range(5):\n",
|
| 553 |
+
" print(f.readline())"
|
| 554 |
+
]
|
| 555 |
+
},
|
| 556 |
+
{
|
| 557 |
+
"cell_type": "code",
|
| 558 |
+
"execution_count": null,
|
| 559 |
+
"id": "7dcde71c-8d46-41e1-938a-e17989f71938",
|
| 560 |
+
"metadata": {},
|
| 561 |
+
"outputs": [],
|
| 562 |
+
"source": []
|
| 563 |
+
},
|
| 564 |
+
{
|
| 565 |
+
"cell_type": "code",
|
| 566 |
+
"execution_count": 22,
|
| 567 |
+
"id": "0d9f7d92-ff3b-4f4c-bb12-c1f973c4c4c0",
|
| 568 |
+
"metadata": {},
|
| 569 |
+
"outputs": [
|
| 570 |
+
{
|
| 571 |
+
"name": "stdout",
|
| 572 |
+
"output_type": "stream",
|
| 573 |
+
"text": [
|
| 574 |
+
"\n",
|
| 575 |
+
"\n",
|
| 576 |
+
"\n"
|
| 577 |
+
]
|
| 578 |
+
}
|
| 579 |
+
],
|
| 580 |
+
"source": [
|
| 581 |
+
"from tokenizers import BertWordPieceTokenizer\n",
|
| 582 |
+
"\n",
|
| 583 |
+
"tokenizer = BertWordPieceTokenizer(lowercase=False) # IMPORTANT for your language\n",
|
| 584 |
+
"\n",
|
| 585 |
+
"tokenizer.train(\n",
|
| 586 |
+
" files=[\"mni_train_tokenise.txt\"],\n",
|
| 587 |
+
" vocab_size=30000,\n",
|
| 588 |
+
" min_frequency=2,\n",
|
| 589 |
+
" special_tokens=[\"[PAD]\", \"[UNK]\", \"[CLS]\", \"[SEP]\", \"[MASK]\"]\n",
|
| 590 |
+
")"
|
| 591 |
+
]
|
| 592 |
+
},
|
| 593 |
+
{
|
| 594 |
+
"cell_type": "code",
|
| 595 |
+
"execution_count": 23,
|
| 596 |
+
"id": "4b0d1138-99a6-45ca-85f6-c8bf518412f0",
|
| 597 |
+
"metadata": {},
|
| 598 |
+
"outputs": [
|
| 599 |
+
{
|
| 600 |
+
"name": "stdout",
|
| 601 |
+
"output_type": "stream",
|
| 602 |
+
"text": [
|
| 603 |
+
"['y', '##o', '##u', '##r', 's', '##a', '##m', '##p', '##l', '##e', 's', '##e', '##n', '##t', '##e', '##n', '##c', '##e']\n"
|
| 604 |
+
]
|
| 605 |
+
}
|
| 606 |
+
],
|
| 607 |
+
"source": [
|
| 608 |
+
"print(tokenizer.encode(\"your sample sentence\").tokens)"
|
| 609 |
+
]
|
| 610 |
+
},
|
| 611 |
+
{
|
| 612 |
+
"cell_type": "code",
|
| 613 |
+
"execution_count": null,
|
| 614 |
+
"id": "6af3fc92-5a19-447b-a85b-330e705f5ee4",
|
| 615 |
+
"metadata": {},
|
| 616 |
+
"outputs": [],
|
| 617 |
+
"source": []
|
| 618 |
+
},
|
| 619 |
+
{
|
| 620 |
+
"cell_type": "code",
|
| 621 |
+
"execution_count": null,
|
| 622 |
+
"id": "40ade035-a0a0-4459-b7aa-56f9ff0e2506",
|
| 623 |
+
"metadata": {},
|
| 624 |
+
"outputs": [],
|
| 625 |
+
"source": []
|
| 626 |
+
},
|
| 627 |
+
{
|
| 628 |
+
"cell_type": "code",
|
| 629 |
+
"execution_count": 19,
|
| 630 |
+
"id": "8629fb3d-a3ca-491d-9323-72a4fd99d52c",
|
| 631 |
+
"metadata": {},
|
| 632 |
+
"outputs": [
|
| 633 |
+
{
|
| 634 |
+
"name": "stderr",
|
| 635 |
+
"output_type": "stream",
|
| 636 |
+
"text": [
|
| 637 |
+
"/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/torch/utils/data/dataloader.py:666: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n",
|
| 638 |
+
" warnings.warn(warn_msg)\n"
|
| 639 |
+
]
|
| 640 |
+
},
|
| 641 |
+
{
|
| 642 |
+
"ename": "IndexError",
|
| 643 |
+
"evalue": "too many indices for array: array is 0-dimensional, but 3 were indexed",
|
| 644 |
+
"output_type": "error",
|
| 645 |
+
"traceback": [
|
| 646 |
+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
| 647 |
+
"\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)",
|
| 648 |
+
"Cell \u001b[0;32mIn[19], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# 11. Train\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# =========================\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
|
| 649 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/transformers/trainer.py:1424\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m 1422\u001b[0m ctx \u001b[38;5;241m=\u001b[39m suppress_progress_bars() \u001b[38;5;28;01mif\u001b[39;00m args\u001b[38;5;241m.\u001b[39mpush_to_hub \u001b[38;5;28;01melse\u001b[39;00m contextlib\u001b[38;5;241m.\u001b[39mnullcontext()\n\u001b[1;32m 1423\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ctx:\n\u001b[0;32m-> 1424\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1425\u001b[0m \u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1426\u001b[0m \u001b[43m \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1427\u001b[0m \u001b[43m \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1428\u001b[0m \u001b[43m \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1429\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
|
| 650 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/transformers/trainer.py:1506\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m 1504\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(epochs_trained, num_train_epochs):\n\u001b[1;32m 1505\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback_handler\u001b[38;5;241m.\u001b[39mon_epoch_begin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol)\n\u001b[0;32m-> 1506\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_run_epoch\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1507\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1508\u001b[0m \u001b[43m \u001b[49m\u001b[43mepoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mepoch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1509\u001b[0m \u001b[43m \u001b[49m\u001b[43mtrain_dataloader\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrain_dataloader\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1510\u001b[0m \u001b[43m \u001b[49m\u001b[43msteps_in_epoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msteps_in_epoch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1511\u001b[0m \u001b[43m \u001b[49m\u001b[43mnum_update_steps_per_epoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_update_steps_per_epoch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1512\u001b[0m \u001b[43m \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1513\u001b[0m \u001b[43m \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1514\u001b[0m \u001b[43m \u001b[49m\u001b[43mstart_time\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstart_time\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1515\u001b[0m \u001b[43m \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1516\u001b[0m \u001b[43m \u001b[49m\u001b[43mepochs_trained\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mepochs_trained\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1517\u001b[0m \u001b[43m \u001b[49m\u001b[43msteps_trained_in_current_epoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msteps_trained_in_current_epoch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1518\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol\u001b[38;5;241m.\u001b[39mshould_training_stop:\n\u001b[1;32m 1520\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n",
|
| 651 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/transformers/trainer.py:1701\u001b[0m, in \u001b[0;36mTrainer._run_epoch\u001b[0;34m(self, model, epoch, train_dataloader, steps_in_epoch, num_update_steps_per_epoch, trial, ignore_keys_for_eval, start_time, resume_from_checkpoint, epochs_trained, steps_trained_in_current_epoch)\u001b[0m\n\u001b[1;32m 1697\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m update_step \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(num_update_steps_trained, num_update_steps_per_epoch):\n\u001b[1;32m 1698\u001b[0m num_batches \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 1699\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mgradient_accumulation_steps \u001b[38;5;28;01mif\u001b[39;00m update_step \u001b[38;5;241m!=\u001b[39m (num_update_steps_per_epoch \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m) \u001b[38;5;28;01melse\u001b[39;00m remainder\n\u001b[1;32m 1700\u001b[0m )\n\u001b[0;32m-> 1701\u001b[0m batch_samples, num_items_in_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_batch_samples\u001b[49m\u001b[43m(\u001b[49m\u001b[43mepoch_iterator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_batches\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1703\u001b[0m \u001b[38;5;66;03m# This is used to correctly scale the loss when the last accumulation step has fewer batches.\u001b[39;00m\n\u001b[1;32m 1704\u001b[0m \u001b[38;5;66;03m# Not used if `num_items_in_batch` is not None.\u001b[39;00m\n\u001b[1;32m 1705\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcurrent_gradient_accumulation_steps \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(batch_samples)\n",
|
| 652 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/transformers/trainer.py:2102\u001b[0m, in \u001b[0;36mTrainer.get_batch_samples\u001b[0;34m(self, epoch_iterator, num_batches, device)\u001b[0m\n\u001b[1;32m 2100\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(num_batches):\n\u001b[1;32m 2101\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 2102\u001b[0m batch_samples\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;28;43mnext\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mepoch_iterator\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 2103\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m 2104\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n",
|
| 653 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/accelerate/data_loader.py:577\u001b[0m, in \u001b[0;36mDataLoaderShard.__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 575\u001b[0m \u001b[38;5;66;03m# We iterate one batch ahead to check when we are at the end\u001b[39;00m\n\u001b[1;32m 576\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 577\u001b[0m current_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mnext\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mdataloader_iter\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 578\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m 579\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mend()\n",
|
| 654 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/torch/utils/data/dataloader.py:734\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 731\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 732\u001b[0m \u001b[38;5;66;03m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[1;32m 733\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset() \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[0;32m--> 734\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 735\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 736\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 737\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable\n\u001b[1;32m 738\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 739\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called\n\u001b[1;32m 740\u001b[0m ):\n",
|
| 655 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/torch/utils/data/dataloader.py:790\u001b[0m, in \u001b[0;36m_SingleProcessDataLoaderIter._next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 788\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_next_data\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 789\u001b[0m index \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_next_index() \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[0;32m--> 790\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset_fetcher\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[1;32m 791\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory:\n\u001b[1;32m 792\u001b[0m data \u001b[38;5;241m=\u001b[39m _utils\u001b[38;5;241m.\u001b[39mpin_memory\u001b[38;5;241m.\u001b[39mpin_memory(data, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory_device)\n",
|
| 656 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/torch/utils/data/_utils/fetch.py:55\u001b[0m, in \u001b[0;36m_MapDatasetFetcher.fetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 54\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n\u001b[0;32m---> 55\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollate_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n",
|
| 657 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/transformers/data/data_collator.py:42\u001b[0m, in \u001b[0;36mDataCollatorMixin.__call__\u001b[0;34m(self, features, return_tensors)\u001b[0m\n\u001b[1;32m 40\u001b[0m return_tensors \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturn_tensors\n\u001b[1;32m 41\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m return_tensors \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m---> 42\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtorch_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeatures\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m return_tensors \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnp\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 44\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnumpy_call(features)\n",
|
| 658 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/transformers/data/data_collator.py:786\u001b[0m, in \u001b[0;36mDataCollatorForLanguageModeling.torch_call\u001b[0;34m(self, examples)\u001b[0m\n\u001b[1;32m 784\u001b[0m offset_mapping \u001b[38;5;241m=\u001b[39m batch\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moffset_mapping\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m 785\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmlm:\n\u001b[0;32m--> 786\u001b[0m batch[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m], batch[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlabels\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtorch_mask_tokens\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 787\u001b[0m \u001b[43m \u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minput_ids\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mspecial_tokens_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mspecial_tokens_mask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moffset_mapping\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moffset_mapping\u001b[49m\n\u001b[1;32m 788\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 789\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 790\u001b[0m labels \u001b[38;5;241m=\u001b[39m batch[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mclone()\n",
|
| 659 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/transformers/data/data_collator.py:813\u001b[0m, in \u001b[0;36mDataCollatorForLanguageModeling.torch_mask_tokens\u001b[0;34m(self, inputs, special_tokens_mask, offset_mapping)\u001b[0m\n\u001b[1;32m 808\u001b[0m special_tokens_mask \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 809\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtokenizer\u001b[38;5;241m.\u001b[39mget_special_tokens_mask(val, already_has_special_tokens\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m) \u001b[38;5;28;01mfor\u001b[39;00m val \u001b[38;5;129;01min\u001b[39;00m labels\u001b[38;5;241m.\u001b[39mtolist()\n\u001b[1;32m 810\u001b[0m ]\n\u001b[1;32m 812\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwhole_word_mask:\n\u001b[0;32m--> 813\u001b[0m word_ids, no_mask_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_calc_word_ids_and_prob_mask\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 814\u001b[0m \u001b[43m \u001b[49m\u001b[43mto_numpy\u001b[49m\u001b[43m(\u001b[49m\u001b[43moffset_mapping\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mto_numpy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mspecial_tokens_mask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 815\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 816\u001b[0m no_mask_mask \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtensor(no_mask_mask, dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mbool)\n\u001b[1;32m 817\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
|
| 660 |
+
"File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.12/site-packages/transformers/data/data_collator.py:982\u001b[0m, in \u001b[0;36mDataCollatorForLanguageModeling._calc_word_ids_and_prob_mask\u001b[0;34m(offsets, special_tokens_mask)\u001b[0m\n\u001b[1;32m 972\u001b[0m \u001b[38;5;129m@staticmethod\u001b[39m\n\u001b[1;32m 973\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_calc_word_ids_and_prob_mask\u001b[39m(\n\u001b[1;32m 974\u001b[0m offsets: np\u001b[38;5;241m.\u001b[39mndarray[np\u001b[38;5;241m.\u001b[39mndarray[\u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;28mint\u001b[39m]]], special_tokens_mask: np\u001b[38;5;241m.\u001b[39mndarray[np\u001b[38;5;241m.\u001b[39mndarray[\u001b[38;5;28mint\u001b[39m]]\n\u001b[1;32m 975\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mtuple\u001b[39m[np\u001b[38;5;241m.\u001b[39mndarray[np\u001b[38;5;241m.\u001b[39mndarray[\u001b[38;5;28mint\u001b[39m]], np\u001b[38;5;241m.\u001b[39mndarray[np\u001b[38;5;241m.\u001b[39mndarray[\u001b[38;5;28mint\u001b[39m]]]:\n\u001b[1;32m 976\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 977\u001b[0m \u001b[38;5;124;03m Map tokens to word ids and create mask of tokens to not mask.\u001b[39;00m\n\u001b[1;32m 978\u001b[0m \u001b[38;5;124;03m Tokens that are part of the same word will have the same word id and we will only\u001b[39;00m\n\u001b[1;32m 979\u001b[0m \u001b[38;5;124;03m set a mask probability for the first token of each word.\u001b[39;00m\n\u001b[1;32m 980\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 982\u001b[0m token_starts \u001b[38;5;241m=\u001b[39m \u001b[43moffsets\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m 983\u001b[0m token_ends \u001b[38;5;241m=\u001b[39m offsets[:, :, \u001b[38;5;241m1\u001b[39m]\n\u001b[1;32m 985\u001b[0m prev_token_ends \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mroll(token_ends, \u001b[38;5;241m1\u001b[39m, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n",
|
| 661 |
+
"\u001b[0;31mIndexError\u001b[0m: too many indices for array: array is 0-dimensional, but 3 were indexed"
|
| 662 |
+
]
|
| 663 |
+
}
|
| 664 |
+
],
|
| 665 |
+
"source": [
|
| 666 |
+
"# 11. Train\n",
|
| 667 |
+
"# =========================\n",
|
| 668 |
+
"trainer.train()\n"
|
| 669 |
+
]
|
| 670 |
+
},
|
| 671 |
+
{
|
| 672 |
+
"cell_type": "code",
|
| 673 |
+
"execution_count": null,
|
| 674 |
+
"id": "02154aea-9daf-49c5-89c0-54838401f3ec",
|
| 675 |
+
"metadata": {},
|
| 676 |
+
"outputs": [],
|
| 677 |
+
"source": []
|
| 678 |
+
},
|
| 679 |
+
{
|
| 680 |
+
"cell_type": "code",
|
| 681 |
+
"execution_count": 5,
|
| 682 |
+
"id": "264f25b3-970c-4384-b062-79c40c3dbaf9",
|
| 683 |
+
"metadata": {},
|
| 684 |
+
"outputs": [],
|
| 685 |
+
"source": [
|
| 686 |
+
"# # =========================\n",
|
| 687 |
+
"# # 9. Training Arguments\n",
|
| 688 |
+
"# # =========================\n",
|
| 689 |
+
"# from transformers import TrainingArguments\n",
|
| 690 |
+
"\n",
|
| 691 |
+
"# training_args = TrainingArguments(\n",
|
| 692 |
+
"# output_dir=\"./bert-wwm\",\n",
|
| 693 |
+
"# overwrite_output_dir=True,\n",
|
| 694 |
+
"# per_device_train_batch_size=8, # reduce if memory error\n",
|
| 695 |
+
"# num_train_epochs=1,\n",
|
| 696 |
+
"# learning_rate=5e-5,\n",
|
| 697 |
+
"# save_steps=5000,\n",
|
| 698 |
+
"# save_total_limit=2,\n",
|
| 699 |
+
"# logging_steps=100,\n",
|
| 700 |
+
"# prediction_loss_only=True,\n",
|
| 701 |
+
"# fp16=True\n",
|
| 702 |
+
"# )"
|
| 703 |
+
]
|
| 704 |
+
},
|
| 705 |
+
{
|
| 706 |
+
"cell_type": "code",
|
| 707 |
+
"execution_count": null,
|
| 708 |
+
"id": "8787b0d9-102a-4767-87ef-9e9a5f10ac18",
|
| 709 |
+
"metadata": {},
|
| 710 |
+
"outputs": [],
|
| 711 |
+
"source": [
|
| 712 |
+
"\n",
|
| 713 |
+
"\n",
|
| 714 |
+
"\n",
|
| 715 |
+
"\n",
|
| 716 |
+
"# =========================\n",
|
| 717 |
+
"# 10. Trainer\n",
|
| 718 |
+
"# =========================\n",
|
| 719 |
+
"from transformers import Trainer\n",
|
| 720 |
+
"\n",
|
| 721 |
+
"trainer = Trainer(\n",
|
| 722 |
+
" model=model,\n",
|
| 723 |
+
" args=training_args,\n",
|
| 724 |
+
" train_dataset=lm_dataset[\"train\"], # IMPORTANT CHANGE\n",
|
| 725 |
+
" data_collator=data_collator\n",
|
| 726 |
+
")\n",
|
| 727 |
+
"\n",
|
| 728 |
+
"# =========================\n",
|
| 729 |
+
"# 11. Train\n",
|
| 730 |
+
"# =========================\n",
|
| 731 |
+
"trainer.train()\n",
|
| 732 |
+
"\n",
|
| 733 |
+
"# =========================\n",
|
| 734 |
+
"# 12. Save Model\n",
|
| 735 |
+
"# =========================\n",
|
| 736 |
+
"trainer.save_model(\"./bert-wwm-final\")\n",
|
| 737 |
+
"tokenizer.save_pretrained(\"./bert-wwm-final\")"
|
| 738 |
+
]
|
| 739 |
+
},
|
| 740 |
+
{
|
| 741 |
+
"cell_type": "code",
|
| 742 |
+
"execution_count": null,
|
| 743 |
+
"id": "8b45757a-8dc9-4011-9158-52d1593c6bae",
|
| 744 |
+
"metadata": {},
|
| 745 |
+
"outputs": [],
|
| 746 |
+
"source": []
|
| 747 |
+
},
|
| 748 |
+
{
|
| 749 |
+
"cell_type": "code",
|
| 750 |
+
"execution_count": null,
|
| 751 |
+
"id": "9f5fb547-dcad-44a5-9de6-a3d6cc6dd882",
|
| 752 |
+
"metadata": {},
|
| 753 |
+
"outputs": [],
|
| 754 |
+
"source": []
|
| 755 |
+
}
|
| 756 |
+
],
|
| 757 |
+
"metadata": {
|
| 758 |
+
"kernelspec": {
|
| 759 |
+
"display_name": "Python 3",
|
| 760 |
+
"language": "python",
|
| 761 |
+
"name": "python3"
|
| 762 |
+
},
|
| 763 |
+
"language_info": {
|
| 764 |
+
"codemirror_mode": {
|
| 765 |
+
"name": "ipython",
|
| 766 |
+
"version": 3
|
| 767 |
+
},
|
| 768 |
+
"file_extension": ".py",
|
| 769 |
+
"mimetype": "text/x-python",
|
| 770 |
+
"name": "python",
|
| 771 |
+
"nbconvert_exporter": "python",
|
| 772 |
+
"pygments_lexer": "ipython3",
|
| 773 |
+
"version": "3.12.11"
|
| 774 |
+
}
|
| 775 |
+
},
|
| 776 |
+
"nbformat": 4,
|
| 777 |
+
"nbformat_minor": 5
|
| 778 |
+
}
|
bart/data/tokenizer/tokenizer.json
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": "1.0",
|
| 3 |
+
"truncation": null,
|
| 4 |
+
"padding": null,
|
| 5 |
+
"added_tokens": [
|
| 6 |
+
{
|
| 7 |
+
"id": 0,
|
| 8 |
+
"content": "[PAD]",
|
| 9 |
+
"single_word": false,
|
| 10 |
+
"lstrip": false,
|
| 11 |
+
"rstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"special": true
|
| 14 |
+
},
|
| 15 |
+
{
|
| 16 |
+
"id": 1,
|
| 17 |
+
"content": "[UNK]",
|
| 18 |
+
"single_word": false,
|
| 19 |
+
"lstrip": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"normalized": false,
|
| 22 |
+
"special": true
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"id": 2,
|
| 26 |
+
"content": "[CLS]",
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"lstrip": false,
|
| 29 |
+
"rstrip": false,
|
| 30 |
+
"normalized": false,
|
| 31 |
+
"special": true
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"id": 3,
|
| 35 |
+
"content": "[SEP]",
|
| 36 |
+
"single_word": false,
|
| 37 |
+
"lstrip": false,
|
| 38 |
+
"rstrip": false,
|
| 39 |
+
"normalized": false,
|
| 40 |
+
"special": true
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"id": 4,
|
| 44 |
+
"content": "[MASK]",
|
| 45 |
+
"single_word": false,
|
| 46 |
+
"lstrip": false,
|
| 47 |
+
"rstrip": false,
|
| 48 |
+
"normalized": false,
|
| 49 |
+
"special": true
|
| 50 |
+
}
|
| 51 |
+
],
|
| 52 |
+
"normalizer": {
|
| 53 |
+
"type": "BertNormalizer",
|
| 54 |
+
"clean_text": true,
|
| 55 |
+
"handle_chinese_chars": true,
|
| 56 |
+
"strip_accents": null,
|
| 57 |
+
"lowercase": true
|
| 58 |
+
},
|
| 59 |
+
"pre_tokenizer": {
|
| 60 |
+
"type": "BertPreTokenizer"
|
| 61 |
+
},
|
| 62 |
+
"post_processor": {
|
| 63 |
+
"type": "TemplateProcessing",
|
| 64 |
+
"single": [
|
| 65 |
+
{
|
| 66 |
+
"SpecialToken": {
|
| 67 |
+
"id": "[CLS]",
|
| 68 |
+
"type_id": 0
|
| 69 |
+
}
|
| 70 |
+
},
|
| 71 |
+
{
|
| 72 |
+
"Sequence": {
|
| 73 |
+
"id": "A",
|
| 74 |
+
"type_id": 0
|
| 75 |
+
}
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"SpecialToken": {
|
| 79 |
+
"id": "[SEP]",
|
| 80 |
+
"type_id": 0
|
| 81 |
+
}
|
| 82 |
+
}
|
| 83 |
+
],
|
| 84 |
+
"pair": [
|
| 85 |
+
{
|
| 86 |
+
"SpecialToken": {
|
| 87 |
+
"id": "[CLS]",
|
| 88 |
+
"type_id": 0
|
| 89 |
+
}
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"Sequence": {
|
| 93 |
+
"id": "A",
|
| 94 |
+
"type_id": 0
|
| 95 |
+
}
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"SpecialToken": {
|
| 99 |
+
"id": "[SEP]",
|
| 100 |
+
"type_id": 0
|
| 101 |
+
}
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"Sequence": {
|
| 105 |
+
"id": "B",
|
| 106 |
+
"type_id": 1
|
| 107 |
+
}
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"SpecialToken": {
|
| 111 |
+
"id": "[SEP]",
|
| 112 |
+
"type_id": 1
|
| 113 |
+
}
|
| 114 |
+
}
|
| 115 |
+
],
|
| 116 |
+
"special_tokens": {
|
| 117 |
+
"[CLS]": {
|
| 118 |
+
"id": "[CLS]",
|
| 119 |
+
"ids": [
|
| 120 |
+
2
|
| 121 |
+
],
|
| 122 |
+
"tokens": [
|
| 123 |
+
"[CLS]"
|
| 124 |
+
]
|
| 125 |
+
},
|
| 126 |
+
"[SEP]": {
|
| 127 |
+
"id": "[SEP]",
|
| 128 |
+
"ids": [
|
| 129 |
+
3
|
| 130 |
+
],
|
| 131 |
+
"tokens": [
|
| 132 |
+
"[SEP]"
|
| 133 |
+
]
|
| 134 |
+
}
|
| 135 |
+
}
|
| 136 |
+
},
|
| 137 |
+
"decoder": {
|
| 138 |
+
"type": "WordPiece",
|
| 139 |
+
"prefix": "##",
|
| 140 |
+
"cleanup": true
|
| 141 |
+
},
|
| 142 |
+
"model": {
|
| 143 |
+
"type": "WordPiece",
|
| 144 |
+
"unk_token": "[UNK]",
|
| 145 |
+
"continuing_subword_prefix": "##",
|
| 146 |
+
"max_input_chars_per_word": 100,
|
| 147 |
+
"vocab": {
|
| 148 |
+
"[PAD]": 0,
|
| 149 |
+
"[UNK]": 1,
|
| 150 |
+
"[CLS]": 2,
|
| 151 |
+
"[SEP]": 3,
|
| 152 |
+
"[MASK]": 4
|
| 153 |
+
}
|
| 154 |
+
}
|
| 155 |
+
}
|
bart/data/tokenizer/tokenizer_config.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"backend": "tokenizers",
|
| 3 |
+
"cls_token": "[CLS]",
|
| 4 |
+
"do_lower_case": true,
|
| 5 |
+
"mask_token": "[MASK]",
|
| 6 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 7 |
+
"pad_token": "[PAD]",
|
| 8 |
+
"sep_token": "[SEP]",
|
| 9 |
+
"strip_accents": null,
|
| 10 |
+
"tokenize_chinese_chars": true,
|
| 11 |
+
"tokenizer_class": "BertTokenizer",
|
| 12 |
+
"unk_token": "[UNK]"
|
| 13 |
+
}
|
bart/data/tokenizer/vocab.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/data-bin/dict.eng_Latn.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/data-bin/dict.mni_Beng.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/data-bin/preprocess.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='data-bin', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='mni_Beng', srcdict=None, suppress_crashes=False, target_lang='eng_Latn', task='translation', tensorboard_logdir=None, testpref='../data/test', tgtdict=None, threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='../data/train', use_plasma_view=False, user_dir=None, validpref='../data/dev', wandb_project=None, workers=1)
|
| 2 |
+
[mni_Beng] Dictionary: 185504 types
|
| 3 |
+
[mni_Beng] ../data/train.mni_Beng: 139266 sents, 2418773 tokens, 0.0% replaced (by <unk>)
|
| 4 |
+
[mni_Beng] Dictionary: 185504 types
|
| 5 |
+
[mni_Beng] ../data/dev.mni_Beng: 2404 sents, 41351 tokens, 3.87% replaced (by <unk>)
|
| 6 |
+
[mni_Beng] Dictionary: 185504 types
|
| 7 |
+
[mni_Beng] ../data/test.mni_Beng: 1183 sents, 20030 tokens, 4.03% replaced (by <unk>)
|
| 8 |
+
[eng_Latn] Dictionary: 104488 types
|
| 9 |
+
[eng_Latn] ../data/train.eng_Latn: 139266 sents, 2851668 tokens, 0.0% replaced (by <unk>)
|
| 10 |
+
[eng_Latn] Dictionary: 104488 types
|
| 11 |
+
[eng_Latn] ../data/dev.eng_Latn: 2404 sents, 48999 tokens, 1.72% replaced (by <unk>)
|
| 12 |
+
[eng_Latn] Dictionary: 104488 types
|
| 13 |
+
[eng_Latn] ../data/test.eng_Latn: 1183 sents, 23873 tokens, 1.63% replaced (by <unk>)
|
| 14 |
+
Wrote preprocessed data to data-bin
|
data/data-bin/test.mni_Beng-eng_Latn.eng_Latn.idx
ADDED
|
Binary file (14.2 kB). View file
|
|
|
data/data-bin/test.mni_Beng-eng_Latn.mni_Beng.idx
ADDED
|
Binary file (14.2 kB). View file
|
|
|
data/data-bin/valid.mni_Beng-eng_Latn.eng_Latn.idx
ADDED
|
Binary file (28.9 kB). View file
|
|
|
data/data-bin/valid.mni_Beng-eng_Latn.mni_Beng.idx
ADDED
|
Binary file (28.9 kB). View file
|
|
|
data/dev.eng_Latn
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/dev.mni_Beng
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/fairseq/.gitignore
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# JetBrains PyCharm IDE
|
| 2 |
+
.idea/
|
| 3 |
+
|
| 4 |
+
# Byte-compiled / optimized / DLL files
|
| 5 |
+
__pycache__/
|
| 6 |
+
*.py[cod]
|
| 7 |
+
*$py.class
|
| 8 |
+
|
| 9 |
+
# C extensions
|
| 10 |
+
*.so
|
| 11 |
+
|
| 12 |
+
# macOS dir files
|
| 13 |
+
.DS_Store
|
| 14 |
+
|
| 15 |
+
# Distribution / packaging
|
| 16 |
+
.Python
|
| 17 |
+
env/
|
| 18 |
+
build/
|
| 19 |
+
develop-eggs/
|
| 20 |
+
dist/
|
| 21 |
+
downloads/
|
| 22 |
+
eggs/
|
| 23 |
+
.eggs/
|
| 24 |
+
lib/
|
| 25 |
+
lib64/
|
| 26 |
+
parts/
|
| 27 |
+
sdist/
|
| 28 |
+
var/
|
| 29 |
+
wheels/
|
| 30 |
+
*.egg-info/
|
| 31 |
+
.installed.cfg
|
| 32 |
+
*.egg
|
| 33 |
+
|
| 34 |
+
# Checkpoints
|
| 35 |
+
checkpoints
|
| 36 |
+
|
| 37 |
+
# PyInstaller
|
| 38 |
+
# Usually these files are written by a python script from a template
|
| 39 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
| 40 |
+
*.manifest
|
| 41 |
+
*.spec
|
| 42 |
+
|
| 43 |
+
# Installer logs
|
| 44 |
+
pip-log.txt
|
| 45 |
+
pip-delete-this-directory.txt
|
| 46 |
+
|
| 47 |
+
# Unit test / coverage reports
|
| 48 |
+
htmlcov/
|
| 49 |
+
.tox/
|
| 50 |
+
.coverage
|
| 51 |
+
.coverage.*
|
| 52 |
+
.cache
|
| 53 |
+
nosetests.xml
|
| 54 |
+
coverage.xml
|
| 55 |
+
*.cover
|
| 56 |
+
.hypothesis/
|
| 57 |
+
|
| 58 |
+
# Translations
|
| 59 |
+
*.mo
|
| 60 |
+
*.pot
|
| 61 |
+
|
| 62 |
+
# Django stuff:
|
| 63 |
+
*.log
|
| 64 |
+
local_settings.py
|
| 65 |
+
|
| 66 |
+
# Flask stuff:
|
| 67 |
+
instance/
|
| 68 |
+
.webassets-cache
|
| 69 |
+
|
| 70 |
+
# Scrapy stuff:
|
| 71 |
+
.scrapy
|
| 72 |
+
|
| 73 |
+
# Sphinx documentation
|
| 74 |
+
docs/_build/
|
| 75 |
+
|
| 76 |
+
# PyBuilder
|
| 77 |
+
target/
|
| 78 |
+
|
| 79 |
+
# Jupyter Notebook
|
| 80 |
+
.ipynb_checkpoints
|
| 81 |
+
|
| 82 |
+
# pyenv
|
| 83 |
+
.python-version
|
| 84 |
+
|
| 85 |
+
# celery beat schedule file
|
| 86 |
+
celerybeat-schedule
|
| 87 |
+
|
| 88 |
+
# SageMath parsed files
|
| 89 |
+
*.sage.py
|
| 90 |
+
|
| 91 |
+
# dotenv
|
| 92 |
+
.env
|
| 93 |
+
|
| 94 |
+
# virtualenv
|
| 95 |
+
.venv
|
| 96 |
+
venv/
|
| 97 |
+
ENV/
|
| 98 |
+
|
| 99 |
+
# Spyder project settings
|
| 100 |
+
.spyderproject
|
| 101 |
+
.spyproject
|
| 102 |
+
|
| 103 |
+
# Rope project settings
|
| 104 |
+
.ropeproject
|
| 105 |
+
|
| 106 |
+
# mkdocs documentation
|
| 107 |
+
/site
|
| 108 |
+
|
| 109 |
+
# mypy
|
| 110 |
+
.mypy_cache/
|
| 111 |
+
|
| 112 |
+
# Generated files
|
| 113 |
+
/fairseq/temporal_convolution_tbc
|
| 114 |
+
/fairseq/modules/*_layer/*_forward.cu
|
| 115 |
+
/fairseq/modules/*_layer/*_backward.cu
|
| 116 |
+
/fairseq/version.py
|
| 117 |
+
|
| 118 |
+
# data
|
| 119 |
+
data-bin/
|
| 120 |
+
|
| 121 |
+
# reranking
|
| 122 |
+
/examples/reranking/rerank_data
|
| 123 |
+
|
| 124 |
+
# Cython-generated C++ source files
|
| 125 |
+
/fairseq/data/data_utils_fast.cpp
|
| 126 |
+
/fairseq/data/token_block_utils_fast.cpp
|
| 127 |
+
|
| 128 |
+
# VSCODE
|
| 129 |
+
.vscode/ftp-sync.json
|
| 130 |
+
.vscode/settings.json
|
| 131 |
+
|
| 132 |
+
# Experimental Folder
|
| 133 |
+
experimental/*
|
| 134 |
+
|
| 135 |
+
# Weights and Biases logs
|
| 136 |
+
wandb/
|
| 137 |
+
|
| 138 |
+
# Hydra artifacts
|
| 139 |
+
nohup.out
|
| 140 |
+
multirun
|
| 141 |
+
outputs
|
data/fairseq/.gitmodules
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[submodule "fairseq/model_parallel/megatron"]
|
| 2 |
+
path = fairseq/model_parallel/megatron
|
| 3 |
+
url = https://github.com/ngoyal2707/Megatron-LM
|
| 4 |
+
branch = fairseq
|
data/fairseq/.pre-commit-config.yaml
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
exclude: 'build|stubs'
|
| 2 |
+
|
| 3 |
+
default_language_version:
|
| 4 |
+
python: python3
|
| 5 |
+
|
| 6 |
+
repos:
|
| 7 |
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
| 8 |
+
rev: v4.1.0
|
| 9 |
+
hooks:
|
| 10 |
+
- id: trailing-whitespace
|
| 11 |
+
- id: check-ast
|
| 12 |
+
- id: check-merge-conflict
|
| 13 |
+
- id: no-commit-to-branch
|
| 14 |
+
args: ['--branch=master']
|
| 15 |
+
- id: check-added-large-files
|
| 16 |
+
args: ['--maxkb=500']
|
| 17 |
+
- id: end-of-file-fixer
|
| 18 |
+
|
| 19 |
+
- repo: https://github.com/ambv/black
|
| 20 |
+
rev: 22.3.0
|
| 21 |
+
hooks:
|
| 22 |
+
- id: black
|
| 23 |
+
language_version: python3.8
|
| 24 |
+
|
| 25 |
+
- repo: https://gitlab.com/pycqa/flake8
|
| 26 |
+
rev: 3.9.2
|
| 27 |
+
hooks:
|
| 28 |
+
- id: flake8
|
| 29 |
+
args: [
|
| 30 |
+
# only error for syntax errors and undefined names
|
| 31 |
+
"--select=E9,F63,F7,F82",
|
| 32 |
+
]
|
| 33 |
+
|
| 34 |
+
- repo: https://github.com/pycqa/isort
|
| 35 |
+
rev: 5.10.1
|
| 36 |
+
hooks:
|
| 37 |
+
- id: isort
|
| 38 |
+
exclude: README.md
|
| 39 |
+
additional_dependencies: [toml]
|
| 40 |
+
args: ["--profile", "black"]
|
data/fairseq/CODE_OF_CONDUCT.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Code of Conduct
|
| 2 |
+
|
| 3 |
+
## Our Pledge
|
| 4 |
+
|
| 5 |
+
In the interest of fostering an open and welcoming environment, we as
|
| 6 |
+
contributors and maintainers pledge to make participation in our project and
|
| 7 |
+
our community a harassment-free experience for everyone, regardless of age, body
|
| 8 |
+
size, disability, ethnicity, sex characteristics, gender identity and expression,
|
| 9 |
+
level of experience, education, socio-economic status, nationality, personal
|
| 10 |
+
appearance, race, religion, or sexual identity and orientation.
|
| 11 |
+
|
| 12 |
+
## Our Standards
|
| 13 |
+
|
| 14 |
+
Examples of behavior that contributes to creating a positive environment
|
| 15 |
+
include:
|
| 16 |
+
|
| 17 |
+
* Using welcoming and inclusive language
|
| 18 |
+
* Being respectful of differing viewpoints and experiences
|
| 19 |
+
* Gracefully accepting constructive criticism
|
| 20 |
+
* Focusing on what is best for the community
|
| 21 |
+
* Showing empathy towards other community members
|
| 22 |
+
|
| 23 |
+
Examples of unacceptable behavior by participants include:
|
| 24 |
+
|
| 25 |
+
* The use of sexualized language or imagery and unwelcome sexual attention or
|
| 26 |
+
advances
|
| 27 |
+
* Trolling, insulting/derogatory comments, and personal or political attacks
|
| 28 |
+
* Public or private harassment
|
| 29 |
+
* Publishing others' private information, such as a physical or electronic
|
| 30 |
+
address, without explicit permission
|
| 31 |
+
* Other conduct which could reasonably be considered inappropriate in a
|
| 32 |
+
professional setting
|
| 33 |
+
|
| 34 |
+
## Our Responsibilities
|
| 35 |
+
|
| 36 |
+
Project maintainers are responsible for clarifying the standards of acceptable
|
| 37 |
+
behavior and are expected to take appropriate and fair corrective action in
|
| 38 |
+
response to any instances of unacceptable behavior.
|
| 39 |
+
|
| 40 |
+
Project maintainers have the right and responsibility to remove, edit, or
|
| 41 |
+
reject comments, commits, code, wiki edits, issues, and other contributions
|
| 42 |
+
that are not aligned to this Code of Conduct, or to ban temporarily or
|
| 43 |
+
permanently any contributor for other behaviors that they deem inappropriate,
|
| 44 |
+
threatening, offensive, or harmful.
|
| 45 |
+
|
| 46 |
+
## Scope
|
| 47 |
+
|
| 48 |
+
This Code of Conduct applies within all project spaces, and it also applies when
|
| 49 |
+
an individual is representing the project or its community in public spaces.
|
| 50 |
+
Examples of representing a project or community include using an official
|
| 51 |
+
project e-mail address, posting via an official social media account, or acting
|
| 52 |
+
as an appointed representative at an online or offline event. Representation of
|
| 53 |
+
a project may be further defined and clarified by project maintainers.
|
| 54 |
+
|
| 55 |
+
## Enforcement
|
| 56 |
+
|
| 57 |
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
| 58 |
+
reported by contacting the project team at <conduct@pytorch.org>. All
|
| 59 |
+
complaints will be reviewed and investigated and will result in a response that
|
| 60 |
+
is deemed necessary and appropriate to the circumstances. The project team is
|
| 61 |
+
obligated to maintain confidentiality with regard to the reporter of an incident.
|
| 62 |
+
Further details of specific enforcement policies may be posted separately.
|
| 63 |
+
|
| 64 |
+
Project maintainers who do not follow or enforce the Code of Conduct in good
|
| 65 |
+
faith may face temporary or permanent repercussions as determined by other
|
| 66 |
+
members of the project's leadership.
|
| 67 |
+
|
| 68 |
+
## Attribution
|
| 69 |
+
|
| 70 |
+
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
| 71 |
+
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
|
| 72 |
+
|
| 73 |
+
[homepage]: https://www.contributor-covenant.org
|
| 74 |
+
|
| 75 |
+
For answers to common questions about this code of conduct, see
|
| 76 |
+
https://www.contributor-covenant.org/faq
|
| 77 |
+
|
data/fairseq/CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Contributing to Facebook AI Research Sequence-to-Sequence Toolkit (fairseq)
|
| 2 |
+
We want to make contributing to this project as easy and transparent as
|
| 3 |
+
possible.
|
| 4 |
+
|
| 5 |
+
## Pull Requests
|
| 6 |
+
We actively welcome your pull requests.
|
| 7 |
+
|
| 8 |
+
1. Fork the repo and create your branch from `main`.
|
| 9 |
+
2. If you've added code that should be tested, add tests.
|
| 10 |
+
3. If you've changed APIs, update the documentation.
|
| 11 |
+
4. Ensure the test suite passes.
|
| 12 |
+
5. Make sure your code lints.
|
| 13 |
+
6. If you haven't already, complete the Contributor License Agreement ("CLA").
|
| 14 |
+
|
| 15 |
+
## Contributor License Agreement ("CLA")
|
| 16 |
+
In order to accept your pull request, we need you to submit a CLA. You only need
|
| 17 |
+
to do this once to work on any of Facebook's open source projects.
|
| 18 |
+
|
| 19 |
+
Complete your CLA here: <https://code.facebook.com/cla>
|
| 20 |
+
|
| 21 |
+
## Issues
|
| 22 |
+
We use GitHub issues to track public bugs. Please ensure your description is
|
| 23 |
+
clear and has sufficient instructions to be able to reproduce the issue.
|
| 24 |
+
|
| 25 |
+
## License
|
| 26 |
+
By contributing to Facebook AI Research Sequence-to-Sequence Toolkit (fairseq),
|
| 27 |
+
you agree that your contributions will be licensed under the LICENSE file in
|
| 28 |
+
the root directory of this source tree.
|
| 29 |
+
|
| 30 |
+
## Pre-commit hooks
|
| 31 |
+
In order to ensure your code lints, there are pre-commit hooks configured in the repository which you can install.
|
| 32 |
+
After installation, they will automatically run each time you commit.
|
| 33 |
+
An abbreviated guide is given below; for more information, refer to [the offical pre-commit documentation](https://pre-commit.com/).
|
| 34 |
+
|
| 35 |
+
### Installation
|
| 36 |
+
```
|
| 37 |
+
pip install pre-commit
|
| 38 |
+
pre-commit install
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
### Usage
|
| 42 |
+
Just commit your changes:
|
| 43 |
+
```
|
| 44 |
+
git commit -m "My informative commit message"
|
| 45 |
+
```
|
| 46 |
+
|
| 47 |
+
If there was a failure, you will get feedback
|
| 48 |
+
```
|
| 49 |
+
[INFO] Initializing environment for https://github.com/PyCQA/flake8.
|
| 50 |
+
[INFO] Installing environment for https://github.com/pre-commit/pre-commit-hooks.
|
| 51 |
+
[INFO] Once installed this environment will be reused.
|
| 52 |
+
[INFO] This may take a few minutes...
|
| 53 |
+
[INFO] Installing environment for https://github.com/PyCQA/flake8.
|
| 54 |
+
[INFO] Once installed this environment will be reused.
|
| 55 |
+
[INFO] This may take a few minutes...
|
| 56 |
+
Trim Trailing Whitespace.................................................Failed
|
| 57 |
+
- hook id: trailing-whitespace
|
| 58 |
+
- exit code: 1
|
| 59 |
+
- files were modified by this hook
|
| 60 |
+
Fixing examples/nllb/modeling/wmt15_benchmark/eval_langs2.sh
|
| 61 |
+
Fix End of Files.........................................................Failed
|
| 62 |
+
- hook id: end-of-file-fixer
|
| 63 |
+
- exit code: 1
|
| 64 |
+
- files were modified by this hook
|
| 65 |
+
Fixing examples/few_shot/scripts/schedule_jobs_few_shot.py
|
| 66 |
+
flake8...................................................................Passed
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
Certain hooks modify your files to comply.
|
| 70 |
+
To include these modifications, you will need to add them (i.e. `git add ...`) and commit again.
|
| 71 |
+
|
| 72 |
+
If all is well, you should see something like:
|
| 73 |
+
```
|
| 74 |
+
Trim Trailing Whitespace.................................................Passed
|
| 75 |
+
Fix End of Files.........................................................Passed
|
| 76 |
+
flake8...................................................................Passed
|
| 77 |
+
[gshard-fix-ci 8698644e1] Fix lint, add pre-commit hooks
|
| 78 |
+
10 files changed, 148 insertions(+), 110 deletions(-)
|
| 79 |
+
create mode 100644 .flake8
|
| 80 |
+
create mode 100644 .pre-commit-config.yaml
|
| 81 |
+
rename examples/nllb/modeling/wmt15_benchmark/{eval_langs2.py => eval_langs2.sh} (99%)
|
| 82 |
+
```
|
data/fairseq/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) Facebook, Inc. and its affiliates.
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
data/fairseq/MANIFEST.in
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
include fairseq/version.txt
|
data/fairseq/README.md
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<p align="center">
|
| 2 |
+
<img src="docs/fairseq_logo.png" width="150">
|
| 3 |
+
<br />
|
| 4 |
+
<br />
|
| 5 |
+
<a href="https://opensource.fb.com/support-ukraine"><img alt="Support Ukraine" src="https://img.shields.io/badge/Support-Ukraine-FFD500?style=flat&labelColor=005BBB" /></a>
|
| 6 |
+
<a href="https://github.com/pytorch/fairseq/blob/main/LICENSE"><img alt="MIT License" src="https://img.shields.io/badge/license-MIT-blue.svg" /></a>
|
| 7 |
+
<a href="https://github.com/pytorch/fairseq/releases"><img alt="Latest Release" src="https://img.shields.io/github/release/pytorch/fairseq.svg" /></a>
|
| 8 |
+
<a href="https://github.com/pytorch/fairseq/actions?query=workflow:build"><img alt="Build Status" src="https://github.com/pytorch/fairseq/workflows/build/badge.svg" /></a>
|
| 9 |
+
<a href="https://fairseq.readthedocs.io/en/latest/?badge=latest"><img alt="Documentation Status" src="https://readthedocs.org/projects/fairseq/badge/?version=latest" /></a>
|
| 10 |
+
<a href="https://app.circleci.com/pipelines/github/facebookresearch/fairseq/"><img alt="CicleCI Status" src="https://circleci.com/gh/facebookresearch/fairseq.svg?style=shield" /></a>
|
| 11 |
+
</p>
|
| 12 |
+
|
| 13 |
+
--------------------------------------------------------------------------------
|
| 14 |
+
|
| 15 |
+
Fairseq(-py) is a sequence modeling toolkit that allows researchers and
|
| 16 |
+
developers to train custom models for translation, summarization, language
|
| 17 |
+
modeling and other text generation tasks.
|
| 18 |
+
|
| 19 |
+
We provide reference implementations of various sequence modeling papers:
|
| 20 |
+
|
| 21 |
+
<details><summary>List of implemented papers</summary><p>
|
| 22 |
+
|
| 23 |
+
* **Convolutional Neural Networks (CNN)**
|
| 24 |
+
+ [Language Modeling with Gated Convolutional Networks (Dauphin et al., 2017)](examples/language_model/conv_lm/README.md)
|
| 25 |
+
+ [Convolutional Sequence to Sequence Learning (Gehring et al., 2017)](examples/conv_seq2seq/README.md)
|
| 26 |
+
+ [Classical Structured Prediction Losses for Sequence to Sequence Learning (Edunov et al., 2018)](https://github.com/pytorch/fairseq/tree/classic_seqlevel)
|
| 27 |
+
+ [Hierarchical Neural Story Generation (Fan et al., 2018)](examples/stories/README.md)
|
| 28 |
+
+ [wav2vec: Unsupervised Pre-training for Speech Recognition (Schneider et al., 2019)](examples/wav2vec/README.md)
|
| 29 |
+
* **LightConv and DynamicConv models**
|
| 30 |
+
+ [Pay Less Attention with Lightweight and Dynamic Convolutions (Wu et al., 2019)](examples/pay_less_attention_paper/README.md)
|
| 31 |
+
* **Long Short-Term Memory (LSTM) networks**
|
| 32 |
+
+ Effective Approaches to Attention-based Neural Machine Translation (Luong et al., 2015)
|
| 33 |
+
* **Transformer (self-attention) networks**
|
| 34 |
+
+ Attention Is All You Need (Vaswani et al., 2017)
|
| 35 |
+
+ [Scaling Neural Machine Translation (Ott et al., 2018)](examples/scaling_nmt/README.md)
|
| 36 |
+
+ [Understanding Back-Translation at Scale (Edunov et al., 2018)](examples/backtranslation/README.md)
|
| 37 |
+
+ [Adaptive Input Representations for Neural Language Modeling (Baevski and Auli, 2018)](examples/language_model/README.adaptive_inputs.md)
|
| 38 |
+
+ [Lexically constrained decoding with dynamic beam allocation (Post & Vilar, 2018)](examples/constrained_decoding/README.md)
|
| 39 |
+
+ [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context (Dai et al., 2019)](examples/truncated_bptt/README.md)
|
| 40 |
+
+ [Adaptive Attention Span in Transformers (Sukhbaatar et al., 2019)](examples/adaptive_span/README.md)
|
| 41 |
+
+ [Mixture Models for Diverse Machine Translation: Tricks of the Trade (Shen et al., 2019)](examples/translation_moe/README.md)
|
| 42 |
+
+ [RoBERTa: A Robustly Optimized BERT Pretraining Approach (Liu et al., 2019)](examples/roberta/README.md)
|
| 43 |
+
+ [Facebook FAIR's WMT19 News Translation Task Submission (Ng et al., 2019)](examples/wmt19/README.md)
|
| 44 |
+
+ [Jointly Learning to Align and Translate with Transformer Models (Garg et al., 2019)](examples/joint_alignment_translation/README.md )
|
| 45 |
+
+ [Multilingual Denoising Pre-training for Neural Machine Translation (Liu et at., 2020)](examples/mbart/README.md)
|
| 46 |
+
+ [Neural Machine Translation with Byte-Level Subwords (Wang et al., 2020)](examples/byte_level_bpe/README.md)
|
| 47 |
+
+ [Unsupervised Quality Estimation for Neural Machine Translation (Fomicheva et al., 2020)](examples/unsupervised_quality_estimation/README.md)
|
| 48 |
+
+ [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations (Baevski et al., 2020)](examples/wav2vec/README.md)
|
| 49 |
+
+ [Generating Medical Reports from Patient-Doctor Conversations Using Sequence-to-Sequence Models (Enarvi et al., 2020)](examples/pointer_generator/README.md)
|
| 50 |
+
+ [Linformer: Self-Attention with Linear Complexity (Wang et al., 2020)](examples/linformer/README.md)
|
| 51 |
+
+ [Cross-lingual Retrieval for Iterative Self-Supervised Training (Tran et al., 2020)](examples/criss/README.md)
|
| 52 |
+
+ [Deep Transformers with Latent Depth (Li et al., 2020)](examples/latent_depth/README.md)
|
| 53 |
+
+ [Unsupervised Cross-lingual Representation Learning for Speech Recognition (Conneau et al., 2020)](https://arxiv.org/abs/2006.13979)
|
| 54 |
+
+ [Self-training and Pre-training are Complementary for Speech Recognition (Xu et al., 2020)](https://arxiv.org/abs/2010.11430)
|
| 55 |
+
+ [Robust wav2vec 2.0: Analyzing Domain Shift in Self-Supervised Pre-Training (Hsu, et al., 2021)](https://arxiv.org/abs/2104.01027)
|
| 56 |
+
+ [Unsupervised Speech Recognition (Baevski, et al., 2021)](https://arxiv.org/abs/2105.11084)
|
| 57 |
+
+ [Simple and Effective Zero-shot Cross-lingual Phoneme Recognition (Xu et al., 2021)](https://arxiv.org/abs/2109.11680)
|
| 58 |
+
+ [VideoCLIP: Contrastive Pre-training for Zero-shot Video-Text Understanding (Xu et. al., 2021)](https://arxiv.org/pdf/2109.14084.pdf)
|
| 59 |
+
+ [VLM: Task-agnostic Video-Language Model Pre-training for Video Understanding (Xu et. al., 2021)](https://aclanthology.org/2021.findings-acl.370.pdf)
|
| 60 |
+
+ [NormFormer: Improved Transformer Pretraining with Extra Normalization (Shleifer et. al, 2021)](examples/normformer/README.md)
|
| 61 |
+
* **Non-autoregressive Transformers**
|
| 62 |
+
+ Non-Autoregressive Neural Machine Translation (Gu et al., 2017)
|
| 63 |
+
+ Deterministic Non-Autoregressive Neural Sequence Modeling by Iterative Refinement (Lee et al. 2018)
|
| 64 |
+
+ Insertion Transformer: Flexible Sequence Generation via Insertion Operations (Stern et al. 2019)
|
| 65 |
+
+ Mask-Predict: Parallel Decoding of Conditional Masked Language Models (Ghazvininejad et al., 2019)
|
| 66 |
+
+ [Levenshtein Transformer (Gu et al., 2019)](examples/nonautoregressive_translation/README.md)
|
| 67 |
+
* **Finetuning**
|
| 68 |
+
+ [Better Fine-Tuning by Reducing Representational Collapse (Aghajanyan et al. 2020)](examples/rxf/README.md)
|
| 69 |
+
|
| 70 |
+
</p></details>
|
| 71 |
+
|
| 72 |
+
### What's New:
|
| 73 |
+
* May 2023 [Released models for Scaling Speech Technology to 1,000+ Languages (Pratap, et al., 2023)](examples/mms/README.md)
|
| 74 |
+
* June 2022 [Released code for wav2vec-U 2.0 from Towards End-to-end Unsupervised Speech Recognition (Liu, et al., 2022)](examples/wav2vec/unsupervised/README.md)
|
| 75 |
+
* May 2022 [Integration with xFormers](https://github.com/facebookresearch/xformers)
|
| 76 |
+
* December 2021 [Released Direct speech-to-speech translation code](examples/speech_to_speech/README.md)
|
| 77 |
+
* October 2021 [Released VideoCLIP and VLM models](examples/MMPT/README.md)
|
| 78 |
+
* October 2021 [Released multilingual finetuned XLSR-53 model](examples/wav2vec/README.md)
|
| 79 |
+
* September 2021 [`master` branch renamed to `main`](https://github.com/github/renaming).
|
| 80 |
+
* July 2021 [Released DrNMT code](examples/discriminative_reranking_nmt/README.md)
|
| 81 |
+
* July 2021 [Released Robust wav2vec 2.0 model](examples/wav2vec/README.md)
|
| 82 |
+
* June 2021 [Released XLMR-XL and XLMR-XXL models](examples/xlmr/README.md)
|
| 83 |
+
* May 2021 [Released Unsupervised Speech Recognition code](examples/wav2vec/unsupervised/README.md)
|
| 84 |
+
* March 2021 [Added full parameter and optimizer state sharding + CPU offloading](examples/fully_sharded_data_parallel/README.md)
|
| 85 |
+
* February 2021 [Added LASER training code](examples/laser/README.md)
|
| 86 |
+
* December 2020: [Added Adaptive Attention Span code](examples/adaptive_span/README.md)
|
| 87 |
+
* December 2020: [GottBERT model and code released](examples/gottbert/README.md)
|
| 88 |
+
* November 2020: Adopted the [Hydra](https://github.com/facebookresearch/hydra) configuration framework
|
| 89 |
+
* [see documentation explaining how to use it for new and existing projects](docs/hydra_integration.md)
|
| 90 |
+
* November 2020: [fairseq 0.10.0 released](https://github.com/pytorch/fairseq/releases/tag/v0.10.0)
|
| 91 |
+
* October 2020: [Added R3F/R4F (Better Fine-Tuning) code](examples/rxf/README.md)
|
| 92 |
+
* October 2020: [Deep Transformer with Latent Depth code released](examples/latent_depth/README.md)
|
| 93 |
+
* October 2020: [Added CRISS models and code](examples/criss/README.md)
|
| 94 |
+
|
| 95 |
+
<details><summary>Previous updates</summary><p>
|
| 96 |
+
|
| 97 |
+
* September 2020: [Added Linformer code](examples/linformer/README.md)
|
| 98 |
+
* September 2020: [Added pointer-generator networks](examples/pointer_generator/README.md)
|
| 99 |
+
* August 2020: [Added lexically constrained decoding](examples/constrained_decoding/README.md)
|
| 100 |
+
* August 2020: [wav2vec2 models and code released](examples/wav2vec/README.md)
|
| 101 |
+
* July 2020: [Unsupervised Quality Estimation code released](examples/unsupervised_quality_estimation/README.md)
|
| 102 |
+
* May 2020: [Follow fairseq on Twitter](https://twitter.com/fairseq)
|
| 103 |
+
* April 2020: [Monotonic Multihead Attention code released](examples/simultaneous_translation/README.md)
|
| 104 |
+
* April 2020: [Quant-Noise code released](examples/quant_noise/README.md)
|
| 105 |
+
* April 2020: [Initial model parallel support and 11B parameters unidirectional LM released](examples/megatron_11b/README.md)
|
| 106 |
+
* March 2020: [Byte-level BPE code released](examples/byte_level_bpe/README.md)
|
| 107 |
+
* February 2020: [mBART model and code released](examples/mbart/README.md)
|
| 108 |
+
* February 2020: [Added tutorial for back-translation](https://github.com/pytorch/fairseq/tree/main/examples/backtranslation#training-your-own-model-wmt18-english-german)
|
| 109 |
+
* December 2019: [fairseq 0.9.0 released](https://github.com/pytorch/fairseq/releases/tag/v0.9.0)
|
| 110 |
+
* November 2019: [VizSeq released (a visual analysis toolkit for evaluating fairseq models)](https://facebookresearch.github.io/vizseq/docs/getting_started/fairseq_example)
|
| 111 |
+
* November 2019: [CamemBERT model and code released](examples/camembert/README.md)
|
| 112 |
+
* November 2019: [BART model and code released](examples/bart/README.md)
|
| 113 |
+
* November 2019: [XLM-R models and code released](examples/xlmr/README.md)
|
| 114 |
+
* September 2019: [Nonautoregressive translation code released](examples/nonautoregressive_translation/README.md)
|
| 115 |
+
* August 2019: [WMT'19 models released](examples/wmt19/README.md)
|
| 116 |
+
* July 2019: fairseq relicensed under MIT license
|
| 117 |
+
* July 2019: [RoBERTa models and code released](examples/roberta/README.md)
|
| 118 |
+
* June 2019: [wav2vec models and code released](examples/wav2vec/README.md)
|
| 119 |
+
|
| 120 |
+
</p></details>
|
| 121 |
+
|
| 122 |
+
### Features:
|
| 123 |
+
|
| 124 |
+
* multi-GPU training on one machine or across multiple machines (data and model parallel)
|
| 125 |
+
* fast generation on both CPU and GPU with multiple search algorithms implemented:
|
| 126 |
+
+ beam search
|
| 127 |
+
+ Diverse Beam Search ([Vijayakumar et al., 2016](https://arxiv.org/abs/1610.02424))
|
| 128 |
+
+ sampling (unconstrained, top-k and top-p/nucleus)
|
| 129 |
+
+ [lexically constrained decoding](examples/constrained_decoding/README.md) (Post & Vilar, 2018)
|
| 130 |
+
* [gradient accumulation](https://fairseq.readthedocs.io/en/latest/getting_started.html#large-mini-batch-training-with-delayed-updates) enables training with large mini-batches even on a single GPU
|
| 131 |
+
* [mixed precision training](https://fairseq.readthedocs.io/en/latest/getting_started.html#training-with-half-precision-floating-point-fp16) (trains faster with less GPU memory on [NVIDIA tensor cores](https://developer.nvidia.com/tensor-cores))
|
| 132 |
+
* [extensible](https://fairseq.readthedocs.io/en/latest/overview.html): easily register new models, criterions, tasks, optimizers and learning rate schedulers
|
| 133 |
+
* [flexible configuration](docs/hydra_integration.md) based on [Hydra](https://github.com/facebookresearch/hydra) allowing a combination of code, command-line and file based configuration
|
| 134 |
+
* [full parameter and optimizer state sharding](examples/fully_sharded_data_parallel/README.md)
|
| 135 |
+
* [offloading parameters to CPU](examples/fully_sharded_data_parallel/README.md)
|
| 136 |
+
|
| 137 |
+
We also provide [pre-trained models for translation and language modeling](#pre-trained-models-and-examples)
|
| 138 |
+
with a convenient `torch.hub` interface:
|
| 139 |
+
|
| 140 |
+
``` python
|
| 141 |
+
en2de = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.en-de.single_model')
|
| 142 |
+
en2de.translate('Hello world', beam=5)
|
| 143 |
+
# 'Hallo Welt'
|
| 144 |
+
```
|
| 145 |
+
|
| 146 |
+
See the PyTorch Hub tutorials for [translation](https://pytorch.org/hub/pytorch_fairseq_translation/)
|
| 147 |
+
and [RoBERTa](https://pytorch.org/hub/pytorch_fairseq_roberta/) for more examples.
|
| 148 |
+
|
| 149 |
+
# Requirements and Installation
|
| 150 |
+
|
| 151 |
+
* [PyTorch](http://pytorch.org/) version >= 1.10.0
|
| 152 |
+
* Python version >= 3.8
|
| 153 |
+
* For training new models, you'll also need an NVIDIA GPU and [NCCL](https://github.com/NVIDIA/nccl)
|
| 154 |
+
* **To install fairseq** and develop locally:
|
| 155 |
+
|
| 156 |
+
``` bash
|
| 157 |
+
git clone https://github.com/pytorch/fairseq
|
| 158 |
+
cd fairseq
|
| 159 |
+
pip install --editable ./
|
| 160 |
+
|
| 161 |
+
# on MacOS:
|
| 162 |
+
# CFLAGS="-stdlib=libc++" pip install --editable ./
|
| 163 |
+
|
| 164 |
+
# to install the latest stable release (0.10.x)
|
| 165 |
+
# pip install fairseq
|
| 166 |
+
```
|
| 167 |
+
|
| 168 |
+
* **For faster training** install NVIDIA's [apex](https://github.com/NVIDIA/apex) library:
|
| 169 |
+
|
| 170 |
+
``` bash
|
| 171 |
+
git clone https://github.com/NVIDIA/apex
|
| 172 |
+
cd apex
|
| 173 |
+
pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" \
|
| 174 |
+
--global-option="--deprecated_fused_adam" --global-option="--xentropy" \
|
| 175 |
+
--global-option="--fast_multihead_attn" ./
|
| 176 |
+
```
|
| 177 |
+
|
| 178 |
+
* **For large datasets** install [PyArrow](https://arrow.apache.org/docs/python/install.html#using-pip): `pip install pyarrow`
|
| 179 |
+
* If you use Docker make sure to increase the shared memory size either with `--ipc=host` or `--shm-size`
|
| 180 |
+
as command line options to `nvidia-docker run` .
|
| 181 |
+
|
| 182 |
+
# Getting Started
|
| 183 |
+
|
| 184 |
+
The [full documentation](https://fairseq.readthedocs.io/) contains instructions
|
| 185 |
+
for getting started, training new models and extending fairseq with new model
|
| 186 |
+
types and tasks.
|
| 187 |
+
|
| 188 |
+
# Pre-trained models and examples
|
| 189 |
+
|
| 190 |
+
We provide pre-trained models and pre-processed, binarized test sets for several tasks listed below,
|
| 191 |
+
as well as example training and evaluation commands.
|
| 192 |
+
|
| 193 |
+
* [Translation](examples/translation/README.md): convolutional and transformer models are available
|
| 194 |
+
* [Language Modeling](examples/language_model/README.md): convolutional and transformer models are available
|
| 195 |
+
|
| 196 |
+
We also have more detailed READMEs to reproduce results from specific papers:
|
| 197 |
+
|
| 198 |
+
* [XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale (Babu et al., 2021)](examples/wav2vec/xlsr/README.md)
|
| 199 |
+
* [Cross-lingual Retrieval for Iterative Self-Supervised Training (Tran et al., 2020)](examples/criss/README.md)
|
| 200 |
+
* [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations (Baevski et al., 2020)](examples/wav2vec/README.md)
|
| 201 |
+
* [Unsupervised Quality Estimation for Neural Machine Translation (Fomicheva et al., 2020)](examples/unsupervised_quality_estimation/README.md)
|
| 202 |
+
* [Training with Quantization Noise for Extreme Model Compression ({Fan*, Stock*} et al., 2020)](examples/quant_noise/README.md)
|
| 203 |
+
* [Neural Machine Translation with Byte-Level Subwords (Wang et al., 2020)](examples/byte_level_bpe/README.md)
|
| 204 |
+
* [Multilingual Denoising Pre-training for Neural Machine Translation (Liu et at., 2020)](examples/mbart/README.md)
|
| 205 |
+
* [Reducing Transformer Depth on Demand with Structured Dropout (Fan et al., 2019)](examples/layerdrop/README.md)
|
| 206 |
+
* [Jointly Learning to Align and Translate with Transformer Models (Garg et al., 2019)](examples/joint_alignment_translation/README.md)
|
| 207 |
+
* [Levenshtein Transformer (Gu et al., 2019)](examples/nonautoregressive_translation/README.md)
|
| 208 |
+
* [Facebook FAIR's WMT19 News Translation Task Submission (Ng et al., 2019)](examples/wmt19/README.md)
|
| 209 |
+
* [RoBERTa: A Robustly Optimized BERT Pretraining Approach (Liu et al., 2019)](examples/roberta/README.md)
|
| 210 |
+
* [wav2vec: Unsupervised Pre-training for Speech Recognition (Schneider et al., 2019)](examples/wav2vec/README.md)
|
| 211 |
+
* [Mixture Models for Diverse Machine Translation: Tricks of the Trade (Shen et al., 2019)](examples/translation_moe/README.md)
|
| 212 |
+
* [Pay Less Attention with Lightweight and Dynamic Convolutions (Wu et al., 2019)](examples/pay_less_attention_paper/README.md)
|
| 213 |
+
* [Understanding Back-Translation at Scale (Edunov et al., 2018)](examples/backtranslation/README.md)
|
| 214 |
+
* [Classical Structured Prediction Losses for Sequence to Sequence Learning (Edunov et al., 2018)](https://github.com/pytorch/fairseq/tree/classic_seqlevel)
|
| 215 |
+
* [Hierarchical Neural Story Generation (Fan et al., 2018)](examples/stories/README.md)
|
| 216 |
+
* [Scaling Neural Machine Translation (Ott et al., 2018)](examples/scaling_nmt/README.md)
|
| 217 |
+
* [Convolutional Sequence to Sequence Learning (Gehring et al., 2017)](examples/conv_seq2seq/README.md)
|
| 218 |
+
* [Language Modeling with Gated Convolutional Networks (Dauphin et al., 2017)](examples/language_model/README.conv.md)
|
| 219 |
+
|
| 220 |
+
# Join the fairseq community
|
| 221 |
+
|
| 222 |
+
* Twitter: https://twitter.com/fairseq
|
| 223 |
+
* Facebook page: https://www.facebook.com/groups/fairseq.users
|
| 224 |
+
* Google group: https://groups.google.com/forum/#!forum/fairseq-users
|
| 225 |
+
|
| 226 |
+
# License
|
| 227 |
+
|
| 228 |
+
fairseq(-py) is MIT-licensed.
|
| 229 |
+
The license applies to the pre-trained models as well.
|
| 230 |
+
|
| 231 |
+
# Citation
|
| 232 |
+
|
| 233 |
+
Please cite as:
|
| 234 |
+
|
| 235 |
+
``` bibtex
|
| 236 |
+
@inproceedings{ott2019fairseq,
|
| 237 |
+
title = {fairseq: A Fast, Extensible Toolkit for Sequence Modeling},
|
| 238 |
+
author = {Myle Ott and Sergey Edunov and Alexei Baevski and Angela Fan and Sam Gross and Nathan Ng and David Grangier and Michael Auli},
|
| 239 |
+
booktitle = {Proceedings of NAACL-HLT 2019: Demonstrations},
|
| 240 |
+
year = {2019},
|
| 241 |
+
}
|
| 242 |
+
```
|
data/fairseq/RELEASE.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Creating a New Release
|
| 2 |
+
|
| 3 |
+
In order to create a new release:
|
| 4 |
+
|
| 5 |
+
1. Navigate to the [Fairseq Workflows](https://github.com/facebookresearch/fairseq/actions) and find the one named _Fairseq Release_.
|
| 6 |
+
|
| 7 |
+
2. Under _Run Workflow_ choose the branch `main` and for _Release Type_ enter either `major`, `minor`, or `patch`.
|
| 8 |
+
|
| 9 |
+
3. A branch named `$new_version-release` will be created where the `version.txt` file is updated. Merge those changes into `main`.
|
| 10 |
+
|
| 11 |
+
4. Make sure that a [new PYPI package](https://pypi.org/project/fairseq/) has been uploaded.
|
| 12 |
+
|
| 13 |
+
5. Make sure that a [new github release](https://github.com/facebookresearch/fairseq/releases) has been created.
|
data/fairseq/fairseq/__pycache__/__init__.cpython-38.pyc
ADDED
|
Binary file (1.18 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/binarizer.cpython-38.pyc
ADDED
|
Binary file (9.64 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/checkpoint_utils.cpython-38.pyc
ADDED
|
Binary file (21.7 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/file_chunker_utils.cpython-38.pyc
ADDED
|
Binary file (2.77 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/file_io.cpython-38.pyc
ADDED
|
Binary file (5.07 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/file_utils.cpython-38.pyc
ADDED
|
Binary file (9 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/hub_utils.cpython-38.pyc
ADDED
|
Binary file (11.2 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/incremental_decoding_utils.cpython-38.pyc
ADDED
|
Binary file (2.21 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/iterative_refinement_generator.cpython-38.pyc
ADDED
|
Binary file (8.75 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/nan_detector.cpython-38.pyc
ADDED
|
Binary file (3.5 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/ngram_repeat_block.cpython-38.pyc
ADDED
|
Binary file (3.77 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/options.cpython-38.pyc
ADDED
|
Binary file (10.7 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/pdb.cpython-38.pyc
ADDED
|
Binary file (1.3 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/quantization_utils.cpython-38.pyc
ADDED
|
Binary file (3.58 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/registry.cpython-38.pyc
ADDED
|
Binary file (2.54 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/search.cpython-38.pyc
ADDED
|
Binary file (24.2 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/sequence_generator.cpython-38.pyc
ADDED
|
Binary file (24.1 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/speech_generator.cpython-38.pyc
ADDED
|
Binary file (11 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/token_generation_constraints.cpython-38.pyc
ADDED
|
Binary file (16.2 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/tokenizer.cpython-38.pyc
ADDED
|
Binary file (376 Bytes). View file
|
|
|
data/fairseq/fairseq/__pycache__/trainer.cpython-38.pyc
ADDED
|
Binary file (37.5 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/utils.cpython-38.pyc
ADDED
|
Binary file (29.8 kB). View file
|
|
|
data/fairseq/fairseq/__pycache__/version.cpython-38.pyc
ADDED
|
Binary file (175 Bytes). View file
|
|
|
data/fairseq/fairseq/dataclass/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from .configs import FairseqDataclass
|
| 7 |
+
from .constants import ChoiceEnum
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
__all__ = [
|
| 11 |
+
"FairseqDataclass",
|
| 12 |
+
"ChoiceEnum",
|
| 13 |
+
]
|
data/fairseq/fairseq/dataclass/__pycache__/__init__.cpython-38.pyc
ADDED
|
Binary file (289 Bytes). View file
|
|
|
data/fairseq/fairseq/dataclass/__pycache__/configs.cpython-38.pyc
ADDED
|
Binary file (31.8 kB). View file
|
|
|
data/fairseq/fairseq/dataclass/__pycache__/constants.cpython-38.pyc
ADDED
|
Binary file (2.26 kB). View file
|
|
|
data/fairseq/fairseq/dataclass/__pycache__/initialize.cpython-38.pyc
ADDED
|
Binary file (1.81 kB). View file
|
|
|