Add files using upload-large-folder tool
Browse files- data/fairseq/fairseq/benchmark/__init__.py +7 -0
- data/fairseq/fairseq/benchmark/__pycache__/__init__.cpython-38.pyc +0 -0
- data/fairseq/fairseq/benchmark/__pycache__/dummy_dataset.cpython-38.pyc +0 -0
- data/fairseq/fairseq/benchmark/__pycache__/dummy_lm.cpython-38.pyc +0 -0
- data/fairseq/fairseq/benchmark/__pycache__/dummy_masked_lm.cpython-38.pyc +0 -0
- data/fairseq/fairseq/benchmark/__pycache__/dummy_model.cpython-38.pyc +0 -0
- data/fairseq/fairseq/benchmark/__pycache__/dummy_mt.cpython-38.pyc +0 -0
- data/fairseq/fairseq/benchmark/benchmark_multihead_attention.py +172 -0
- data/fairseq/fairseq/benchmark/dummy_dataset.py +36 -0
- data/fairseq/fairseq/benchmark/dummy_lm.py +83 -0
- data/fairseq/fairseq/benchmark/dummy_masked_lm.py +94 -0
- data/fairseq/fairseq/benchmark/dummy_model.py +96 -0
- data/fairseq/fairseq/benchmark/dummy_mt.py +119 -0
- data/fairseq/fairseq/model_parallel/criterions/__init__.py +14 -0
- data/fairseq/fairseq/model_parallel/criterions/__pycache__/__init__.cpython-38.pyc +0 -0
- data/fairseq/fairseq/model_parallel/criterions/__pycache__/vocab_parallel_cross_entropy.cpython-38.pyc +0 -0
- data/fairseq/fairseq/model_parallel/criterions/vocab_parallel_cross_entropy.py +88 -0
- data/fairseq/fairseq/model_parallel/modules/__pycache__/__init__.cpython-38.pyc +0 -0
- data/fairseq/fairseq/model_parallel/modules/__pycache__/multihead_attention.cpython-38.pyc +0 -0
- data/fairseq/fairseq/model_parallel/modules/__pycache__/transformer_layer.cpython-38.pyc +0 -0
data/fairseq/fairseq/benchmark/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
# import models/tasks to register them
|
| 7 |
+
from . import dummy_dataset, dummy_lm, dummy_masked_lm, dummy_model, dummy_mt # noqa
|
data/fairseq/fairseq/benchmark/__pycache__/__init__.cpython-38.pyc
ADDED
|
Binary file (288 Bytes). View file
|
|
|
data/fairseq/fairseq/benchmark/__pycache__/dummy_dataset.cpython-38.pyc
ADDED
|
Binary file (1.77 kB). View file
|
|
|
data/fairseq/fairseq/benchmark/__pycache__/dummy_lm.cpython-38.pyc
ADDED
|
Binary file (3.17 kB). View file
|
|
|
data/fairseq/fairseq/benchmark/__pycache__/dummy_masked_lm.cpython-38.pyc
ADDED
|
Binary file (3.38 kB). View file
|
|
|
data/fairseq/fairseq/benchmark/__pycache__/dummy_model.cpython-38.pyc
ADDED
|
Binary file (3.42 kB). View file
|
|
|
data/fairseq/fairseq/benchmark/__pycache__/dummy_mt.cpython-38.pyc
ADDED
|
Binary file (4.57 kB). View file
|
|
|
data/fairseq/fairseq/benchmark/benchmark_multihead_attention.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import itertools
|
| 7 |
+
import random
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
from torch.utils import benchmark
|
| 11 |
+
|
| 12 |
+
from fairseq.modules.multihead_attention import MultiheadAttention
|
| 13 |
+
|
| 14 |
+
BATCH = [20, 41, 97]
|
| 15 |
+
SEQ = 64
|
| 16 |
+
EMB = 48
|
| 17 |
+
HEADS = 4
|
| 18 |
+
DROP = 0.1
|
| 19 |
+
DEVICE = torch.device("cuda")
|
| 20 |
+
ATTN_MASK_DTYPE = [torch.uint8, torch.bool, torch.float]
|
| 21 |
+
KEY_PADDING_MASK_DTYPE = [torch.uint8, torch.bool]
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _reset_seeds():
|
| 25 |
+
torch.manual_seed(0)
|
| 26 |
+
random.seed(0)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _get_mask(to_dtype: torch.dtype, dim0: int, dim1: int):
|
| 30 |
+
if to_dtype == torch.float:
|
| 31 |
+
mask = torch.randint(0, 2, (dim0, dim1)).to(dtype=torch.bool)
|
| 32 |
+
return mask.to(dtype=to_dtype).masked_fill(mask, -float("inf"))
|
| 33 |
+
return torch.randint(0, 2, (dim0, dim1)).to(dtype=to_dtype)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def benchmark_multihead_attention(
|
| 37 |
+
label="",
|
| 38 |
+
attn_dtype=torch.uint8,
|
| 39 |
+
key_padding_dtype=torch.uint8,
|
| 40 |
+
add_bias_kv=False,
|
| 41 |
+
add_zero_attn=False,
|
| 42 |
+
static_kv=False,
|
| 43 |
+
batch_size=20,
|
| 44 |
+
embedding=EMB,
|
| 45 |
+
seq_len=SEQ,
|
| 46 |
+
num_heads=HEADS,
|
| 47 |
+
):
|
| 48 |
+
|
| 49 |
+
results = []
|
| 50 |
+
# device = torch.device("cuda")
|
| 51 |
+
|
| 52 |
+
xformers_att_config = '{"name": "scaled_dot_product"}'
|
| 53 |
+
|
| 54 |
+
attn_mask = _get_mask(to_dtype=attn_dtype, dim0=seq_len, dim1=seq_len)
|
| 55 |
+
key_padding_mask = _get_mask(
|
| 56 |
+
to_dtype=key_padding_dtype, dim0=batch_size, dim1=seq_len
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
q = torch.rand(seq_len, batch_size, embedding, requires_grad=True)
|
| 60 |
+
k = torch.rand(seq_len, batch_size, embedding, requires_grad=True)
|
| 61 |
+
v = torch.rand(seq_len, batch_size, embedding, requires_grad=True)
|
| 62 |
+
|
| 63 |
+
_reset_seeds()
|
| 64 |
+
|
| 65 |
+
original_mha = MultiheadAttention(
|
| 66 |
+
embedding,
|
| 67 |
+
num_heads,
|
| 68 |
+
dropout=0.0,
|
| 69 |
+
xformers_att_config=None,
|
| 70 |
+
add_bias_kv=add_bias_kv,
|
| 71 |
+
add_zero_attn=add_zero_attn,
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
xformers_mha = MultiheadAttention(
|
| 75 |
+
embedding,
|
| 76 |
+
num_heads,
|
| 77 |
+
dropout=0.0,
|
| 78 |
+
xformers_att_config=xformers_att_config,
|
| 79 |
+
add_bias_kv=add_bias_kv,
|
| 80 |
+
add_zero_attn=add_zero_attn,
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
def original_bench_fw(q, k, v, key_padding_mask, attn_mask, static_kv):
|
| 84 |
+
original_mha(
|
| 85 |
+
query=q,
|
| 86 |
+
key=k,
|
| 87 |
+
value=v,
|
| 88 |
+
key_padding_mask=key_padding_mask,
|
| 89 |
+
attn_mask=attn_mask,
|
| 90 |
+
static_kv=static_kv,
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
def xformers_bench_fw(q, k, v, key_padding_mask, attn_mask, static_kv):
|
| 94 |
+
xformers_mha(
|
| 95 |
+
query=q,
|
| 96 |
+
key=k,
|
| 97 |
+
value=v,
|
| 98 |
+
key_padding_mask=key_padding_mask,
|
| 99 |
+
attn_mask=attn_mask,
|
| 100 |
+
static_kv=static_kv,
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
def original_bench_fw_bw(q, k, v, key_padding_mask, attn_mask, static_kv):
|
| 104 |
+
output, _ = original_mha(
|
| 105 |
+
query=q,
|
| 106 |
+
key=k,
|
| 107 |
+
value=v,
|
| 108 |
+
key_padding_mask=key_padding_mask,
|
| 109 |
+
attn_mask=attn_mask,
|
| 110 |
+
static_kv=static_kv,
|
| 111 |
+
)
|
| 112 |
+
loss = torch.norm(output)
|
| 113 |
+
loss.backward()
|
| 114 |
+
|
| 115 |
+
def xformers_bench_fw_bw(q, k, v, key_padding_mask, attn_mask, static_kv):
|
| 116 |
+
output, _ = xformers_mha(
|
| 117 |
+
query=q,
|
| 118 |
+
key=k,
|
| 119 |
+
value=v,
|
| 120 |
+
key_padding_mask=key_padding_mask,
|
| 121 |
+
attn_mask=attn_mask,
|
| 122 |
+
static_kv=static_kv,
|
| 123 |
+
)
|
| 124 |
+
loss = torch.norm(output)
|
| 125 |
+
loss.backward()
|
| 126 |
+
|
| 127 |
+
fns = [
|
| 128 |
+
original_bench_fw,
|
| 129 |
+
xformers_bench_fw,
|
| 130 |
+
original_bench_fw_bw,
|
| 131 |
+
xformers_bench_fw_bw,
|
| 132 |
+
]
|
| 133 |
+
|
| 134 |
+
for fn in fns:
|
| 135 |
+
results.append(
|
| 136 |
+
benchmark.Timer(
|
| 137 |
+
stmt="fn(q, k, v, key_padding_mask, attn_mask, static_kv)",
|
| 138 |
+
globals={
|
| 139 |
+
"q": q,
|
| 140 |
+
"k": k,
|
| 141 |
+
"v": v,
|
| 142 |
+
"key_padding_mask": key_padding_mask,
|
| 143 |
+
"attn_mask": attn_mask,
|
| 144 |
+
"static_kv": static_kv,
|
| 145 |
+
"fn": fn,
|
| 146 |
+
},
|
| 147 |
+
label="multihead fw + bw",
|
| 148 |
+
sub_label=f"{fn.__name__}",
|
| 149 |
+
description=label,
|
| 150 |
+
).blocked_autorange(min_run_time=1)
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
compare = benchmark.Compare(results)
|
| 154 |
+
compare.print()
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def run_benchmarks():
|
| 158 |
+
for attn_dtype, key_padding_dtype, add_bias_kv, add_zero_attn in itertools.product(
|
| 159 |
+
ATTN_MASK_DTYPE, KEY_PADDING_MASK_DTYPE, [True, False], [True, False]
|
| 160 |
+
):
|
| 161 |
+
label = f"attn_dtype {attn_dtype}, key_padding_dtype {key_padding_dtype}, \
|
| 162 |
+
add_bias_kv {add_bias_kv}, add_zero_attn {add_zero_attn}"
|
| 163 |
+
benchmark_multihead_attention(
|
| 164 |
+
label=label,
|
| 165 |
+
attn_dtype=attn_dtype,
|
| 166 |
+
key_padding_dtype=key_padding_dtype,
|
| 167 |
+
add_bias_kv=add_bias_kv,
|
| 168 |
+
add_zero_attn=add_zero_attn,
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
run_benchmarks()
|
data/fairseq/fairseq/benchmark/dummy_dataset.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
from fairseq.data import FairseqDataset
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class DummyDataset(FairseqDataset):
|
| 6 |
+
def __init__(self, batch, num_items, item_size):
|
| 7 |
+
super().__init__()
|
| 8 |
+
self.batch = batch
|
| 9 |
+
self.num_items = num_items
|
| 10 |
+
self.item_size = item_size
|
| 11 |
+
|
| 12 |
+
def __getitem__(self, index):
|
| 13 |
+
return index
|
| 14 |
+
|
| 15 |
+
def __len__(self):
|
| 16 |
+
return self.num_items
|
| 17 |
+
|
| 18 |
+
def collater(self, samples):
|
| 19 |
+
return self.batch
|
| 20 |
+
|
| 21 |
+
@property
|
| 22 |
+
def sizes(self):
|
| 23 |
+
return np.array([self.item_size] * self.num_items)
|
| 24 |
+
|
| 25 |
+
def num_tokens(self, index):
|
| 26 |
+
return self.item_size
|
| 27 |
+
|
| 28 |
+
def size(self, index):
|
| 29 |
+
return self.item_size
|
| 30 |
+
|
| 31 |
+
def ordered_indices(self):
|
| 32 |
+
return np.arange(self.num_items)
|
| 33 |
+
|
| 34 |
+
@property
|
| 35 |
+
def supports_prefetch(self):
|
| 36 |
+
return False
|
data/fairseq/fairseq/benchmark/dummy_lm.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
from typing import Optional
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
from .dummy_dataset import DummyDataset
|
| 12 |
+
from fairseq.data import Dictionary
|
| 13 |
+
from fairseq.dataclass import FairseqDataclass
|
| 14 |
+
from fairseq.tasks import FairseqTask, register_task
|
| 15 |
+
from omegaconf import II
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass
|
| 22 |
+
class DummyLMConfig(FairseqDataclass):
|
| 23 |
+
dict_size: int = 49996
|
| 24 |
+
dataset_size: int = 100000
|
| 25 |
+
tokens_per_sample: int = field(
|
| 26 |
+
default=512, metadata={"help": "max sequence length"}
|
| 27 |
+
)
|
| 28 |
+
add_bos_token: bool = False
|
| 29 |
+
batch_size: Optional[int] = II("dataset.batch_size")
|
| 30 |
+
max_tokens: Optional[int] = II("dataset.max_tokens")
|
| 31 |
+
max_target_positions: int = II("task.tokens_per_sample")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@register_task("dummy_lm", dataclass=DummyLMConfig)
|
| 35 |
+
class DummyLMTask(FairseqTask):
|
| 36 |
+
def __init__(self, cfg: DummyLMConfig):
|
| 37 |
+
super().__init__(cfg)
|
| 38 |
+
|
| 39 |
+
# load dictionary
|
| 40 |
+
self.dictionary = Dictionary()
|
| 41 |
+
for i in range(cfg.dict_size):
|
| 42 |
+
self.dictionary.add_symbol("word{}".format(i))
|
| 43 |
+
self.dictionary.pad_to_multiple_(8) # often faster if divisible by 8
|
| 44 |
+
logger.info("dictionary: {} types".format(len(self.dictionary)))
|
| 45 |
+
|
| 46 |
+
seq = torch.arange(cfg.tokens_per_sample + 1) + self.dictionary.pad() + 1
|
| 47 |
+
|
| 48 |
+
self.dummy_src = seq[:-1]
|
| 49 |
+
self.dummy_tgt = seq[1:]
|
| 50 |
+
|
| 51 |
+
def load_dataset(self, split, epoch=1, combine=False, **kwargs):
|
| 52 |
+
"""Load a given dataset split.
|
| 53 |
+
Args:
|
| 54 |
+
split (str): name of the split (e.g., train, valid, test)
|
| 55 |
+
"""
|
| 56 |
+
if self.cfg.batch_size is not None:
|
| 57 |
+
bsz = self.cfg.batch_size
|
| 58 |
+
else:
|
| 59 |
+
bsz = max(1, self.cfg.max_tokens // self.cfg.tokens_per_sample)
|
| 60 |
+
self.datasets[split] = DummyDataset(
|
| 61 |
+
{
|
| 62 |
+
"id": 1,
|
| 63 |
+
"net_input": {
|
| 64 |
+
"src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]),
|
| 65 |
+
"src_lengths": torch.full(
|
| 66 |
+
(bsz,), self.cfg.tokens_per_sample, dtype=torch.long
|
| 67 |
+
),
|
| 68 |
+
},
|
| 69 |
+
"target": torch.stack([self.dummy_tgt for _ in range(bsz)]),
|
| 70 |
+
"nsentences": bsz,
|
| 71 |
+
"ntokens": bsz * self.cfg.tokens_per_sample,
|
| 72 |
+
},
|
| 73 |
+
num_items=self.cfg.dataset_size,
|
| 74 |
+
item_size=self.cfg.tokens_per_sample,
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
@property
|
| 78 |
+
def source_dictionary(self):
|
| 79 |
+
return self.dictionary
|
| 80 |
+
|
| 81 |
+
@property
|
| 82 |
+
def target_dictionary(self):
|
| 83 |
+
return self.dictionary
|
data/fairseq/fairseq/benchmark/dummy_masked_lm.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
from typing import Optional
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
from omegaconf import II
|
| 12 |
+
|
| 13 |
+
from .dummy_dataset import DummyDataset
|
| 14 |
+
from fairseq.data import Dictionary
|
| 15 |
+
from fairseq.dataclass import FairseqDataclass
|
| 16 |
+
from fairseq.tasks import FairseqTask, register_task
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass
|
| 22 |
+
class DummyMaskedLMConfig(FairseqDataclass):
|
| 23 |
+
dict_size: int = 49996
|
| 24 |
+
dataset_size: int = 100000
|
| 25 |
+
tokens_per_sample: int = field(
|
| 26 |
+
default=512,
|
| 27 |
+
metadata={
|
| 28 |
+
"help": "max number of total tokens over all"
|
| 29 |
+
" segments per sample for BERT dataset"
|
| 30 |
+
},
|
| 31 |
+
)
|
| 32 |
+
batch_size: Optional[int] = II("dataset.batch_size")
|
| 33 |
+
max_tokens: Optional[int] = II("dataset.max_tokens")
|
| 34 |
+
max_target_positions: int = II("task.tokens_per_sample")
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
@register_task("dummy_masked_lm", dataclass=DummyMaskedLMConfig)
|
| 38 |
+
class DummyMaskedLMTask(FairseqTask):
|
| 39 |
+
def __init__(self, cfg: DummyMaskedLMConfig):
|
| 40 |
+
super().__init__(cfg)
|
| 41 |
+
|
| 42 |
+
self.dictionary = Dictionary()
|
| 43 |
+
for i in range(cfg.dict_size):
|
| 44 |
+
self.dictionary.add_symbol("word{}".format(i))
|
| 45 |
+
logger.info("dictionary: {} types".format(len(self.dictionary)))
|
| 46 |
+
# add mask token
|
| 47 |
+
self.mask_idx = self.dictionary.add_symbol("<mask>")
|
| 48 |
+
self.dictionary.pad_to_multiple_(8) # often faster if divisible by 8
|
| 49 |
+
|
| 50 |
+
mask_idx = 0
|
| 51 |
+
pad_idx = 1
|
| 52 |
+
seq = torch.arange(cfg.tokens_per_sample) + pad_idx + 1
|
| 53 |
+
mask = torch.arange(2, cfg.tokens_per_sample, 7) # ~15%
|
| 54 |
+
src = seq.clone()
|
| 55 |
+
src[mask] = mask_idx
|
| 56 |
+
tgt = torch.full_like(seq, pad_idx)
|
| 57 |
+
tgt[mask] = seq[mask]
|
| 58 |
+
|
| 59 |
+
self.dummy_src = src
|
| 60 |
+
self.dummy_tgt = tgt
|
| 61 |
+
|
| 62 |
+
def load_dataset(self, split, epoch=1, combine=False, **kwargs):
|
| 63 |
+
"""Load a given dataset split.
|
| 64 |
+
Args:
|
| 65 |
+
split (str): name of the split (e.g., train, valid, test)
|
| 66 |
+
"""
|
| 67 |
+
if self.cfg.batch_size is not None:
|
| 68 |
+
bsz = self.cfg.batch_size
|
| 69 |
+
else:
|
| 70 |
+
bsz = max(1, self.cfg.max_tokens // self.cfg.tokens_per_sample)
|
| 71 |
+
self.datasets[split] = DummyDataset(
|
| 72 |
+
{
|
| 73 |
+
"id": 1,
|
| 74 |
+
"net_input": {
|
| 75 |
+
"src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]),
|
| 76 |
+
"src_lengths": torch.full(
|
| 77 |
+
(bsz,), self.cfg.tokens_per_sample, dtype=torch.long
|
| 78 |
+
),
|
| 79 |
+
},
|
| 80 |
+
"target": torch.stack([self.dummy_tgt for _ in range(bsz)]),
|
| 81 |
+
"nsentences": bsz,
|
| 82 |
+
"ntokens": bsz * self.cfg.tokens_per_sample,
|
| 83 |
+
},
|
| 84 |
+
num_items=self.cfg.dataset_size,
|
| 85 |
+
item_size=self.cfg.tokens_per_sample,
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
@property
|
| 89 |
+
def source_dictionary(self):
|
| 90 |
+
return self.dictionary
|
| 91 |
+
|
| 92 |
+
@property
|
| 93 |
+
def target_dictionary(self):
|
| 94 |
+
return self.dictionary
|
data/fairseq/fairseq/benchmark/dummy_model.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
import torch.nn.functional as F
|
| 8 |
+
from fairseq.data import Dictionary
|
| 9 |
+
from fairseq.models import (
|
| 10 |
+
FairseqDecoder,
|
| 11 |
+
FairseqLanguageModel,
|
| 12 |
+
register_model,
|
| 13 |
+
register_model_architecture,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@register_model("dummy_model")
|
| 18 |
+
class DummyModel(FairseqLanguageModel):
|
| 19 |
+
def __init__(self, args, encoder):
|
| 20 |
+
super().__init__(encoder)
|
| 21 |
+
self.args = args
|
| 22 |
+
|
| 23 |
+
@staticmethod
|
| 24 |
+
def add_args(parser):
|
| 25 |
+
parser.add_argument("--num-layers", type=int, default=24)
|
| 26 |
+
parser.add_argument("--embed-dim", type=int, default=1024)
|
| 27 |
+
|
| 28 |
+
@classmethod
|
| 29 |
+
def build_model(cls, args, task):
|
| 30 |
+
encoder = DummyEncoder(
|
| 31 |
+
num_embed=len(task.target_dictionary),
|
| 32 |
+
embed_dim=args.embed_dim,
|
| 33 |
+
num_layers=args.num_layers,
|
| 34 |
+
)
|
| 35 |
+
return cls(args, encoder)
|
| 36 |
+
|
| 37 |
+
def forward(self, src_tokens, masked_tokens=None, **kwargs):
|
| 38 |
+
return self.decoder(src_tokens, masked_tokens=masked_tokens)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class DummyEncoder(FairseqDecoder):
|
| 42 |
+
def __init__(self, num_embed=50000, embed_dim=1024, num_layers=24):
|
| 43 |
+
super().__init__(Dictionary())
|
| 44 |
+
self.embed = nn.Embedding(
|
| 45 |
+
num_embeddings=num_embed, embedding_dim=embed_dim, padding_idx=0
|
| 46 |
+
)
|
| 47 |
+
self.layers_a = nn.ModuleList(
|
| 48 |
+
[
|
| 49 |
+
nn.Sequential(
|
| 50 |
+
nn.LayerNorm(embed_dim),
|
| 51 |
+
nn.Linear(embed_dim, 3 * embed_dim), # q, k, v input projection
|
| 52 |
+
nn.Linear(3 * embed_dim, embed_dim), # skip self-attention
|
| 53 |
+
nn.Linear(embed_dim, embed_dim), # output projection
|
| 54 |
+
nn.Dropout(),
|
| 55 |
+
)
|
| 56 |
+
for i in range(num_layers)
|
| 57 |
+
]
|
| 58 |
+
)
|
| 59 |
+
self.layers_b = nn.ModuleList(
|
| 60 |
+
[
|
| 61 |
+
nn.Sequential(
|
| 62 |
+
nn.LayerNorm(embed_dim),
|
| 63 |
+
nn.Linear(embed_dim, 4 * embed_dim), # FFN
|
| 64 |
+
nn.ReLU(),
|
| 65 |
+
nn.Linear(4 * embed_dim, embed_dim), # FFN
|
| 66 |
+
nn.Dropout(0.1),
|
| 67 |
+
)
|
| 68 |
+
for i in range(num_layers)
|
| 69 |
+
]
|
| 70 |
+
)
|
| 71 |
+
self.out_proj = nn.Linear(embed_dim, num_embed)
|
| 72 |
+
|
| 73 |
+
def forward(self, tokens, masked_tokens=None):
|
| 74 |
+
x = self.embed(tokens)
|
| 75 |
+
for layer_a, layer_b in zip(self.layers_a, self.layers_b):
|
| 76 |
+
x = x + layer_a(x)
|
| 77 |
+
x = x + layer_b(x)
|
| 78 |
+
x = self.out_proj(x)
|
| 79 |
+
if masked_tokens is not None:
|
| 80 |
+
x = x[masked_tokens]
|
| 81 |
+
return (x,)
|
| 82 |
+
|
| 83 |
+
def max_positions(self):
|
| 84 |
+
return 1024
|
| 85 |
+
|
| 86 |
+
def get_normalized_probs(self, net_output, log_probs, sample=None):
|
| 87 |
+
logits = net_output[0].float()
|
| 88 |
+
if log_probs:
|
| 89 |
+
return F.log_softmax(logits, dim=-1)
|
| 90 |
+
else:
|
| 91 |
+
return F.softmax(logits, dim=-1)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
@register_model_architecture("dummy_model", "dummy_model")
|
| 95 |
+
def base_architecture(args):
|
| 96 |
+
pass
|
data/fairseq/fairseq/benchmark/dummy_mt.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
from fairseq.data import Dictionary, FairseqDataset
|
| 12 |
+
from fairseq.tasks import LegacyFairseqTask, register_task
|
| 13 |
+
|
| 14 |
+
logger = logging.getLogger(__name__)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@register_task("dummy_mt")
|
| 18 |
+
class DummyMTTask(LegacyFairseqTask):
|
| 19 |
+
@staticmethod
|
| 20 |
+
def add_args(parser):
|
| 21 |
+
"""Add task-specific arguments to the parser."""
|
| 22 |
+
parser.add_argument("--dict-size", default=49996, type=int)
|
| 23 |
+
parser.add_argument("--dataset-size", default=100000, type=int)
|
| 24 |
+
parser.add_argument("--src-len", default=30, type=int)
|
| 25 |
+
parser.add_argument("--tgt-len", default=30, type=int)
|
| 26 |
+
|
| 27 |
+
def __init__(self, args, dictionary):
|
| 28 |
+
super().__init__(args)
|
| 29 |
+
self.dictionary = dictionary
|
| 30 |
+
self.seed = args.seed
|
| 31 |
+
|
| 32 |
+
dictionary.pad_to_multiple_(8) # often faster if divisible by 8
|
| 33 |
+
|
| 34 |
+
self.dummy_src = torch.arange(args.src_len + 1) + dictionary.pad() + 1
|
| 35 |
+
self.dummy_tgt = torch.arange(args.tgt_len + 1) + dictionary.pad() + 1
|
| 36 |
+
|
| 37 |
+
@classmethod
|
| 38 |
+
def setup_task(cls, args, **kwargs):
|
| 39 |
+
"""Setup the task."""
|
| 40 |
+
dictionary = Dictionary()
|
| 41 |
+
for i in range(args.dict_size):
|
| 42 |
+
dictionary.add_symbol("word{}".format(i))
|
| 43 |
+
logger.info("dictionary: {} types".format(len(dictionary)))
|
| 44 |
+
|
| 45 |
+
args.max_source_positions = args.src_len + dictionary.pad() + 2
|
| 46 |
+
args.max_target_positions = args.tgt_len + dictionary.pad() + 2
|
| 47 |
+
|
| 48 |
+
return cls(args, dictionary)
|
| 49 |
+
|
| 50 |
+
def load_dataset(self, split, epoch=1, combine=False, **kwargs):
|
| 51 |
+
"""Load a given dataset split.
|
| 52 |
+
Args:
|
| 53 |
+
split (str): name of the split (e.g., train, valid, test)
|
| 54 |
+
"""
|
| 55 |
+
item_size = max(self.args.src_len, self.args.tgt_len)
|
| 56 |
+
if self.args.batch_size is not None:
|
| 57 |
+
bsz = self.args.batch_size
|
| 58 |
+
else:
|
| 59 |
+
bsz = max(1, self.args.max_tokens // item_size)
|
| 60 |
+
tgt = torch.stack([self.dummy_tgt for _ in range(bsz)])
|
| 61 |
+
self.datasets[split] = DummyDataset(
|
| 62 |
+
{
|
| 63 |
+
"id": 1,
|
| 64 |
+
"net_input": {
|
| 65 |
+
"src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]),
|
| 66 |
+
"src_lengths": torch.full(
|
| 67 |
+
(bsz,), self.args.src_len, dtype=torch.long
|
| 68 |
+
),
|
| 69 |
+
"prev_output_tokens": tgt.clone(),
|
| 70 |
+
},
|
| 71 |
+
"target": tgt,
|
| 72 |
+
"nsentences": bsz,
|
| 73 |
+
"ntokens": bsz * self.args.tgt_len,
|
| 74 |
+
},
|
| 75 |
+
num_items=self.args.dataset_size,
|
| 76 |
+
item_size=item_size,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def source_dictionary(self):
|
| 81 |
+
return self.dictionary
|
| 82 |
+
|
| 83 |
+
@property
|
| 84 |
+
def target_dictionary(self):
|
| 85 |
+
return self.dictionary
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class DummyDataset(FairseqDataset):
|
| 89 |
+
def __init__(self, batch, num_items, item_size):
|
| 90 |
+
super().__init__()
|
| 91 |
+
self.batch = batch
|
| 92 |
+
self.num_items = num_items
|
| 93 |
+
self.item_size = item_size
|
| 94 |
+
|
| 95 |
+
def __getitem__(self, index):
|
| 96 |
+
return index
|
| 97 |
+
|
| 98 |
+
def __len__(self):
|
| 99 |
+
return self.num_items
|
| 100 |
+
|
| 101 |
+
def collater(self, samples):
|
| 102 |
+
return self.batch
|
| 103 |
+
|
| 104 |
+
@property
|
| 105 |
+
def sizes(self):
|
| 106 |
+
return np.array([self.item_size] * self.num_items)
|
| 107 |
+
|
| 108 |
+
def num_tokens(self, index):
|
| 109 |
+
return self.item_size
|
| 110 |
+
|
| 111 |
+
def size(self, index):
|
| 112 |
+
return self.item_size
|
| 113 |
+
|
| 114 |
+
def ordered_indices(self):
|
| 115 |
+
return np.arange(self.num_items)
|
| 116 |
+
|
| 117 |
+
@property
|
| 118 |
+
def supports_prefetch(self):
|
| 119 |
+
return False
|
data/fairseq/fairseq/model_parallel/criterions/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import importlib
|
| 7 |
+
import os
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
# automatically import any Python files in the criterions/ directory
|
| 11 |
+
for file in sorted(os.listdir(os.path.dirname(__file__))):
|
| 12 |
+
if file.endswith(".py") and not file.startswith("_"):
|
| 13 |
+
module = file[: file.find(".py")]
|
| 14 |
+
importlib.import_module("fairseq.model_parallel.criterions." + module)
|
data/fairseq/fairseq/model_parallel/criterions/__pycache__/__init__.cpython-38.pyc
ADDED
|
Binary file (446 Bytes). View file
|
|
|
data/fairseq/fairseq/model_parallel/criterions/__pycache__/vocab_parallel_cross_entropy.cpython-38.pyc
ADDED
|
Binary file (3.6 kB). View file
|
|
|
data/fairseq/fairseq/model_parallel/criterions/vocab_parallel_cross_entropy.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
|
| 8 |
+
from fairseq import utils
|
| 9 |
+
from fairseq.logging import metrics
|
| 10 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
try:
|
| 14 |
+
from fairseq.model_parallel.megatron.mpu.cross_entropy import (
|
| 15 |
+
vocab_parallel_cross_entropy,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
has_megatron_submodule = True
|
| 19 |
+
except (ImportError, ModuleNotFoundError):
|
| 20 |
+
has_megatron_submodule = False
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@register_criterion("vocab_parallel_cross_entropy")
|
| 24 |
+
class VocabParallelCrossEntropyCriterion(FairseqCriterion):
|
| 25 |
+
def __init__(self, task, sentence_avg):
|
| 26 |
+
super().__init__(task)
|
| 27 |
+
self.sentence_avg = sentence_avg
|
| 28 |
+
if not has_megatron_submodule:
|
| 29 |
+
raise ImportError(
|
| 30 |
+
"\n\nPlease install the megatron submodule:"
|
| 31 |
+
"\n\n git submodule update --init "
|
| 32 |
+
"fairseq/model_parallel/megatron"
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
def forward(self, model, sample, reduce=True):
|
| 36 |
+
"""Compute the loss for the given sample.
|
| 37 |
+
|
| 38 |
+
Returns a tuple with three elements:
|
| 39 |
+
1) the loss
|
| 40 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 41 |
+
3) logging outputs to display while training
|
| 42 |
+
"""
|
| 43 |
+
net_output = model(**sample["net_input"])
|
| 44 |
+
target = sample["target"]
|
| 45 |
+
|
| 46 |
+
loss = vocab_parallel_cross_entropy(net_output[0].float(), target)
|
| 47 |
+
loss = (loss * (target != self.padding_idx)).sum()
|
| 48 |
+
sample_size = (
|
| 49 |
+
sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
|
| 50 |
+
)
|
| 51 |
+
logging_output = {
|
| 52 |
+
"loss": utils.item(loss.data) if reduce else loss.data,
|
| 53 |
+
"ntokens": sample["ntokens"],
|
| 54 |
+
"nsentences": sample["target"].size(0),
|
| 55 |
+
"sample_size": sample_size,
|
| 56 |
+
}
|
| 57 |
+
return loss, sample_size, logging_output
|
| 58 |
+
|
| 59 |
+
@staticmethod
|
| 60 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 61 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 62 |
+
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
|
| 63 |
+
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
|
| 64 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 65 |
+
|
| 66 |
+
metrics.log_scalar(
|
| 67 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 68 |
+
)
|
| 69 |
+
if sample_size != ntokens:
|
| 70 |
+
metrics.log_scalar(
|
| 71 |
+
"nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
|
| 72 |
+
)
|
| 73 |
+
metrics.log_derived(
|
| 74 |
+
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
|
| 75 |
+
)
|
| 76 |
+
else:
|
| 77 |
+
metrics.log_derived(
|
| 78 |
+
"ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
@staticmethod
|
| 82 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 83 |
+
"""
|
| 84 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 85 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 86 |
+
to True will improves distributed training speed.
|
| 87 |
+
"""
|
| 88 |
+
return True
|
data/fairseq/fairseq/model_parallel/modules/__pycache__/__init__.cpython-38.pyc
ADDED
|
Binary file (451 Bytes). View file
|
|
|
data/fairseq/fairseq/model_parallel/modules/__pycache__/multihead_attention.cpython-38.pyc
ADDED
|
Binary file (7.16 kB). View file
|
|
|
data/fairseq/fairseq/model_parallel/modules/__pycache__/transformer_layer.cpython-38.pyc
ADDED
|
Binary file (2.9 kB). View file
|
|
|