sonithoiningombam commited on
Commit
cfc4dda
·
verified ·
1 Parent(s): 94ac172

Add files using upload-large-folder tool

Browse files
data/fairseq/fairseq/benchmark/__init__.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ # import models/tasks to register them
7
+ from . import dummy_dataset, dummy_lm, dummy_masked_lm, dummy_model, dummy_mt # noqa
data/fairseq/fairseq/benchmark/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (288 Bytes). View file
 
data/fairseq/fairseq/benchmark/__pycache__/dummy_dataset.cpython-38.pyc ADDED
Binary file (1.77 kB). View file
 
data/fairseq/fairseq/benchmark/__pycache__/dummy_lm.cpython-38.pyc ADDED
Binary file (3.17 kB). View file
 
data/fairseq/fairseq/benchmark/__pycache__/dummy_masked_lm.cpython-38.pyc ADDED
Binary file (3.38 kB). View file
 
data/fairseq/fairseq/benchmark/__pycache__/dummy_model.cpython-38.pyc ADDED
Binary file (3.42 kB). View file
 
data/fairseq/fairseq/benchmark/__pycache__/dummy_mt.cpython-38.pyc ADDED
Binary file (4.57 kB). View file
 
data/fairseq/fairseq/benchmark/benchmark_multihead_attention.py ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import itertools
7
+ import random
8
+
9
+ import torch
10
+ from torch.utils import benchmark
11
+
12
+ from fairseq.modules.multihead_attention import MultiheadAttention
13
+
14
+ BATCH = [20, 41, 97]
15
+ SEQ = 64
16
+ EMB = 48
17
+ HEADS = 4
18
+ DROP = 0.1
19
+ DEVICE = torch.device("cuda")
20
+ ATTN_MASK_DTYPE = [torch.uint8, torch.bool, torch.float]
21
+ KEY_PADDING_MASK_DTYPE = [torch.uint8, torch.bool]
22
+
23
+
24
+ def _reset_seeds():
25
+ torch.manual_seed(0)
26
+ random.seed(0)
27
+
28
+
29
+ def _get_mask(to_dtype: torch.dtype, dim0: int, dim1: int):
30
+ if to_dtype == torch.float:
31
+ mask = torch.randint(0, 2, (dim0, dim1)).to(dtype=torch.bool)
32
+ return mask.to(dtype=to_dtype).masked_fill(mask, -float("inf"))
33
+ return torch.randint(0, 2, (dim0, dim1)).to(dtype=to_dtype)
34
+
35
+
36
+ def benchmark_multihead_attention(
37
+ label="",
38
+ attn_dtype=torch.uint8,
39
+ key_padding_dtype=torch.uint8,
40
+ add_bias_kv=False,
41
+ add_zero_attn=False,
42
+ static_kv=False,
43
+ batch_size=20,
44
+ embedding=EMB,
45
+ seq_len=SEQ,
46
+ num_heads=HEADS,
47
+ ):
48
+
49
+ results = []
50
+ # device = torch.device("cuda")
51
+
52
+ xformers_att_config = '{"name": "scaled_dot_product"}'
53
+
54
+ attn_mask = _get_mask(to_dtype=attn_dtype, dim0=seq_len, dim1=seq_len)
55
+ key_padding_mask = _get_mask(
56
+ to_dtype=key_padding_dtype, dim0=batch_size, dim1=seq_len
57
+ )
58
+
59
+ q = torch.rand(seq_len, batch_size, embedding, requires_grad=True)
60
+ k = torch.rand(seq_len, batch_size, embedding, requires_grad=True)
61
+ v = torch.rand(seq_len, batch_size, embedding, requires_grad=True)
62
+
63
+ _reset_seeds()
64
+
65
+ original_mha = MultiheadAttention(
66
+ embedding,
67
+ num_heads,
68
+ dropout=0.0,
69
+ xformers_att_config=None,
70
+ add_bias_kv=add_bias_kv,
71
+ add_zero_attn=add_zero_attn,
72
+ )
73
+
74
+ xformers_mha = MultiheadAttention(
75
+ embedding,
76
+ num_heads,
77
+ dropout=0.0,
78
+ xformers_att_config=xformers_att_config,
79
+ add_bias_kv=add_bias_kv,
80
+ add_zero_attn=add_zero_attn,
81
+ )
82
+
83
+ def original_bench_fw(q, k, v, key_padding_mask, attn_mask, static_kv):
84
+ original_mha(
85
+ query=q,
86
+ key=k,
87
+ value=v,
88
+ key_padding_mask=key_padding_mask,
89
+ attn_mask=attn_mask,
90
+ static_kv=static_kv,
91
+ )
92
+
93
+ def xformers_bench_fw(q, k, v, key_padding_mask, attn_mask, static_kv):
94
+ xformers_mha(
95
+ query=q,
96
+ key=k,
97
+ value=v,
98
+ key_padding_mask=key_padding_mask,
99
+ attn_mask=attn_mask,
100
+ static_kv=static_kv,
101
+ )
102
+
103
+ def original_bench_fw_bw(q, k, v, key_padding_mask, attn_mask, static_kv):
104
+ output, _ = original_mha(
105
+ query=q,
106
+ key=k,
107
+ value=v,
108
+ key_padding_mask=key_padding_mask,
109
+ attn_mask=attn_mask,
110
+ static_kv=static_kv,
111
+ )
112
+ loss = torch.norm(output)
113
+ loss.backward()
114
+
115
+ def xformers_bench_fw_bw(q, k, v, key_padding_mask, attn_mask, static_kv):
116
+ output, _ = xformers_mha(
117
+ query=q,
118
+ key=k,
119
+ value=v,
120
+ key_padding_mask=key_padding_mask,
121
+ attn_mask=attn_mask,
122
+ static_kv=static_kv,
123
+ )
124
+ loss = torch.norm(output)
125
+ loss.backward()
126
+
127
+ fns = [
128
+ original_bench_fw,
129
+ xformers_bench_fw,
130
+ original_bench_fw_bw,
131
+ xformers_bench_fw_bw,
132
+ ]
133
+
134
+ for fn in fns:
135
+ results.append(
136
+ benchmark.Timer(
137
+ stmt="fn(q, k, v, key_padding_mask, attn_mask, static_kv)",
138
+ globals={
139
+ "q": q,
140
+ "k": k,
141
+ "v": v,
142
+ "key_padding_mask": key_padding_mask,
143
+ "attn_mask": attn_mask,
144
+ "static_kv": static_kv,
145
+ "fn": fn,
146
+ },
147
+ label="multihead fw + bw",
148
+ sub_label=f"{fn.__name__}",
149
+ description=label,
150
+ ).blocked_autorange(min_run_time=1)
151
+ )
152
+
153
+ compare = benchmark.Compare(results)
154
+ compare.print()
155
+
156
+
157
+ def run_benchmarks():
158
+ for attn_dtype, key_padding_dtype, add_bias_kv, add_zero_attn in itertools.product(
159
+ ATTN_MASK_DTYPE, KEY_PADDING_MASK_DTYPE, [True, False], [True, False]
160
+ ):
161
+ label = f"attn_dtype {attn_dtype}, key_padding_dtype {key_padding_dtype}, \
162
+ add_bias_kv {add_bias_kv}, add_zero_attn {add_zero_attn}"
163
+ benchmark_multihead_attention(
164
+ label=label,
165
+ attn_dtype=attn_dtype,
166
+ key_padding_dtype=key_padding_dtype,
167
+ add_bias_kv=add_bias_kv,
168
+ add_zero_attn=add_zero_attn,
169
+ )
170
+
171
+
172
+ run_benchmarks()
data/fairseq/fairseq/benchmark/dummy_dataset.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ from fairseq.data import FairseqDataset
3
+
4
+
5
+ class DummyDataset(FairseqDataset):
6
+ def __init__(self, batch, num_items, item_size):
7
+ super().__init__()
8
+ self.batch = batch
9
+ self.num_items = num_items
10
+ self.item_size = item_size
11
+
12
+ def __getitem__(self, index):
13
+ return index
14
+
15
+ def __len__(self):
16
+ return self.num_items
17
+
18
+ def collater(self, samples):
19
+ return self.batch
20
+
21
+ @property
22
+ def sizes(self):
23
+ return np.array([self.item_size] * self.num_items)
24
+
25
+ def num_tokens(self, index):
26
+ return self.item_size
27
+
28
+ def size(self, index):
29
+ return self.item_size
30
+
31
+ def ordered_indices(self):
32
+ return np.arange(self.num_items)
33
+
34
+ @property
35
+ def supports_prefetch(self):
36
+ return False
data/fairseq/fairseq/benchmark/dummy_lm.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import logging
7
+ from dataclasses import dataclass, field
8
+ from typing import Optional
9
+
10
+ import torch
11
+ from .dummy_dataset import DummyDataset
12
+ from fairseq.data import Dictionary
13
+ from fairseq.dataclass import FairseqDataclass
14
+ from fairseq.tasks import FairseqTask, register_task
15
+ from omegaconf import II
16
+
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+
21
+ @dataclass
22
+ class DummyLMConfig(FairseqDataclass):
23
+ dict_size: int = 49996
24
+ dataset_size: int = 100000
25
+ tokens_per_sample: int = field(
26
+ default=512, metadata={"help": "max sequence length"}
27
+ )
28
+ add_bos_token: bool = False
29
+ batch_size: Optional[int] = II("dataset.batch_size")
30
+ max_tokens: Optional[int] = II("dataset.max_tokens")
31
+ max_target_positions: int = II("task.tokens_per_sample")
32
+
33
+
34
+ @register_task("dummy_lm", dataclass=DummyLMConfig)
35
+ class DummyLMTask(FairseqTask):
36
+ def __init__(self, cfg: DummyLMConfig):
37
+ super().__init__(cfg)
38
+
39
+ # load dictionary
40
+ self.dictionary = Dictionary()
41
+ for i in range(cfg.dict_size):
42
+ self.dictionary.add_symbol("word{}".format(i))
43
+ self.dictionary.pad_to_multiple_(8) # often faster if divisible by 8
44
+ logger.info("dictionary: {} types".format(len(self.dictionary)))
45
+
46
+ seq = torch.arange(cfg.tokens_per_sample + 1) + self.dictionary.pad() + 1
47
+
48
+ self.dummy_src = seq[:-1]
49
+ self.dummy_tgt = seq[1:]
50
+
51
+ def load_dataset(self, split, epoch=1, combine=False, **kwargs):
52
+ """Load a given dataset split.
53
+ Args:
54
+ split (str): name of the split (e.g., train, valid, test)
55
+ """
56
+ if self.cfg.batch_size is not None:
57
+ bsz = self.cfg.batch_size
58
+ else:
59
+ bsz = max(1, self.cfg.max_tokens // self.cfg.tokens_per_sample)
60
+ self.datasets[split] = DummyDataset(
61
+ {
62
+ "id": 1,
63
+ "net_input": {
64
+ "src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]),
65
+ "src_lengths": torch.full(
66
+ (bsz,), self.cfg.tokens_per_sample, dtype=torch.long
67
+ ),
68
+ },
69
+ "target": torch.stack([self.dummy_tgt for _ in range(bsz)]),
70
+ "nsentences": bsz,
71
+ "ntokens": bsz * self.cfg.tokens_per_sample,
72
+ },
73
+ num_items=self.cfg.dataset_size,
74
+ item_size=self.cfg.tokens_per_sample,
75
+ )
76
+
77
+ @property
78
+ def source_dictionary(self):
79
+ return self.dictionary
80
+
81
+ @property
82
+ def target_dictionary(self):
83
+ return self.dictionary
data/fairseq/fairseq/benchmark/dummy_masked_lm.py ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import logging
7
+ from dataclasses import dataclass, field
8
+ from typing import Optional
9
+
10
+ import torch
11
+ from omegaconf import II
12
+
13
+ from .dummy_dataset import DummyDataset
14
+ from fairseq.data import Dictionary
15
+ from fairseq.dataclass import FairseqDataclass
16
+ from fairseq.tasks import FairseqTask, register_task
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+
21
+ @dataclass
22
+ class DummyMaskedLMConfig(FairseqDataclass):
23
+ dict_size: int = 49996
24
+ dataset_size: int = 100000
25
+ tokens_per_sample: int = field(
26
+ default=512,
27
+ metadata={
28
+ "help": "max number of total tokens over all"
29
+ " segments per sample for BERT dataset"
30
+ },
31
+ )
32
+ batch_size: Optional[int] = II("dataset.batch_size")
33
+ max_tokens: Optional[int] = II("dataset.max_tokens")
34
+ max_target_positions: int = II("task.tokens_per_sample")
35
+
36
+
37
+ @register_task("dummy_masked_lm", dataclass=DummyMaskedLMConfig)
38
+ class DummyMaskedLMTask(FairseqTask):
39
+ def __init__(self, cfg: DummyMaskedLMConfig):
40
+ super().__init__(cfg)
41
+
42
+ self.dictionary = Dictionary()
43
+ for i in range(cfg.dict_size):
44
+ self.dictionary.add_symbol("word{}".format(i))
45
+ logger.info("dictionary: {} types".format(len(self.dictionary)))
46
+ # add mask token
47
+ self.mask_idx = self.dictionary.add_symbol("<mask>")
48
+ self.dictionary.pad_to_multiple_(8) # often faster if divisible by 8
49
+
50
+ mask_idx = 0
51
+ pad_idx = 1
52
+ seq = torch.arange(cfg.tokens_per_sample) + pad_idx + 1
53
+ mask = torch.arange(2, cfg.tokens_per_sample, 7) # ~15%
54
+ src = seq.clone()
55
+ src[mask] = mask_idx
56
+ tgt = torch.full_like(seq, pad_idx)
57
+ tgt[mask] = seq[mask]
58
+
59
+ self.dummy_src = src
60
+ self.dummy_tgt = tgt
61
+
62
+ def load_dataset(self, split, epoch=1, combine=False, **kwargs):
63
+ """Load a given dataset split.
64
+ Args:
65
+ split (str): name of the split (e.g., train, valid, test)
66
+ """
67
+ if self.cfg.batch_size is not None:
68
+ bsz = self.cfg.batch_size
69
+ else:
70
+ bsz = max(1, self.cfg.max_tokens // self.cfg.tokens_per_sample)
71
+ self.datasets[split] = DummyDataset(
72
+ {
73
+ "id": 1,
74
+ "net_input": {
75
+ "src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]),
76
+ "src_lengths": torch.full(
77
+ (bsz,), self.cfg.tokens_per_sample, dtype=torch.long
78
+ ),
79
+ },
80
+ "target": torch.stack([self.dummy_tgt for _ in range(bsz)]),
81
+ "nsentences": bsz,
82
+ "ntokens": bsz * self.cfg.tokens_per_sample,
83
+ },
84
+ num_items=self.cfg.dataset_size,
85
+ item_size=self.cfg.tokens_per_sample,
86
+ )
87
+
88
+ @property
89
+ def source_dictionary(self):
90
+ return self.dictionary
91
+
92
+ @property
93
+ def target_dictionary(self):
94
+ return self.dictionary
data/fairseq/fairseq/benchmark/dummy_model.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import torch.nn as nn
7
+ import torch.nn.functional as F
8
+ from fairseq.data import Dictionary
9
+ from fairseq.models import (
10
+ FairseqDecoder,
11
+ FairseqLanguageModel,
12
+ register_model,
13
+ register_model_architecture,
14
+ )
15
+
16
+
17
+ @register_model("dummy_model")
18
+ class DummyModel(FairseqLanguageModel):
19
+ def __init__(self, args, encoder):
20
+ super().__init__(encoder)
21
+ self.args = args
22
+
23
+ @staticmethod
24
+ def add_args(parser):
25
+ parser.add_argument("--num-layers", type=int, default=24)
26
+ parser.add_argument("--embed-dim", type=int, default=1024)
27
+
28
+ @classmethod
29
+ def build_model(cls, args, task):
30
+ encoder = DummyEncoder(
31
+ num_embed=len(task.target_dictionary),
32
+ embed_dim=args.embed_dim,
33
+ num_layers=args.num_layers,
34
+ )
35
+ return cls(args, encoder)
36
+
37
+ def forward(self, src_tokens, masked_tokens=None, **kwargs):
38
+ return self.decoder(src_tokens, masked_tokens=masked_tokens)
39
+
40
+
41
+ class DummyEncoder(FairseqDecoder):
42
+ def __init__(self, num_embed=50000, embed_dim=1024, num_layers=24):
43
+ super().__init__(Dictionary())
44
+ self.embed = nn.Embedding(
45
+ num_embeddings=num_embed, embedding_dim=embed_dim, padding_idx=0
46
+ )
47
+ self.layers_a = nn.ModuleList(
48
+ [
49
+ nn.Sequential(
50
+ nn.LayerNorm(embed_dim),
51
+ nn.Linear(embed_dim, 3 * embed_dim), # q, k, v input projection
52
+ nn.Linear(3 * embed_dim, embed_dim), # skip self-attention
53
+ nn.Linear(embed_dim, embed_dim), # output projection
54
+ nn.Dropout(),
55
+ )
56
+ for i in range(num_layers)
57
+ ]
58
+ )
59
+ self.layers_b = nn.ModuleList(
60
+ [
61
+ nn.Sequential(
62
+ nn.LayerNorm(embed_dim),
63
+ nn.Linear(embed_dim, 4 * embed_dim), # FFN
64
+ nn.ReLU(),
65
+ nn.Linear(4 * embed_dim, embed_dim), # FFN
66
+ nn.Dropout(0.1),
67
+ )
68
+ for i in range(num_layers)
69
+ ]
70
+ )
71
+ self.out_proj = nn.Linear(embed_dim, num_embed)
72
+
73
+ def forward(self, tokens, masked_tokens=None):
74
+ x = self.embed(tokens)
75
+ for layer_a, layer_b in zip(self.layers_a, self.layers_b):
76
+ x = x + layer_a(x)
77
+ x = x + layer_b(x)
78
+ x = self.out_proj(x)
79
+ if masked_tokens is not None:
80
+ x = x[masked_tokens]
81
+ return (x,)
82
+
83
+ def max_positions(self):
84
+ return 1024
85
+
86
+ def get_normalized_probs(self, net_output, log_probs, sample=None):
87
+ logits = net_output[0].float()
88
+ if log_probs:
89
+ return F.log_softmax(logits, dim=-1)
90
+ else:
91
+ return F.softmax(logits, dim=-1)
92
+
93
+
94
+ @register_model_architecture("dummy_model", "dummy_model")
95
+ def base_architecture(args):
96
+ pass
data/fairseq/fairseq/benchmark/dummy_mt.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import logging
7
+
8
+ import numpy as np
9
+ import torch
10
+
11
+ from fairseq.data import Dictionary, FairseqDataset
12
+ from fairseq.tasks import LegacyFairseqTask, register_task
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ @register_task("dummy_mt")
18
+ class DummyMTTask(LegacyFairseqTask):
19
+ @staticmethod
20
+ def add_args(parser):
21
+ """Add task-specific arguments to the parser."""
22
+ parser.add_argument("--dict-size", default=49996, type=int)
23
+ parser.add_argument("--dataset-size", default=100000, type=int)
24
+ parser.add_argument("--src-len", default=30, type=int)
25
+ parser.add_argument("--tgt-len", default=30, type=int)
26
+
27
+ def __init__(self, args, dictionary):
28
+ super().__init__(args)
29
+ self.dictionary = dictionary
30
+ self.seed = args.seed
31
+
32
+ dictionary.pad_to_multiple_(8) # often faster if divisible by 8
33
+
34
+ self.dummy_src = torch.arange(args.src_len + 1) + dictionary.pad() + 1
35
+ self.dummy_tgt = torch.arange(args.tgt_len + 1) + dictionary.pad() + 1
36
+
37
+ @classmethod
38
+ def setup_task(cls, args, **kwargs):
39
+ """Setup the task."""
40
+ dictionary = Dictionary()
41
+ for i in range(args.dict_size):
42
+ dictionary.add_symbol("word{}".format(i))
43
+ logger.info("dictionary: {} types".format(len(dictionary)))
44
+
45
+ args.max_source_positions = args.src_len + dictionary.pad() + 2
46
+ args.max_target_positions = args.tgt_len + dictionary.pad() + 2
47
+
48
+ return cls(args, dictionary)
49
+
50
+ def load_dataset(self, split, epoch=1, combine=False, **kwargs):
51
+ """Load a given dataset split.
52
+ Args:
53
+ split (str): name of the split (e.g., train, valid, test)
54
+ """
55
+ item_size = max(self.args.src_len, self.args.tgt_len)
56
+ if self.args.batch_size is not None:
57
+ bsz = self.args.batch_size
58
+ else:
59
+ bsz = max(1, self.args.max_tokens // item_size)
60
+ tgt = torch.stack([self.dummy_tgt for _ in range(bsz)])
61
+ self.datasets[split] = DummyDataset(
62
+ {
63
+ "id": 1,
64
+ "net_input": {
65
+ "src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]),
66
+ "src_lengths": torch.full(
67
+ (bsz,), self.args.src_len, dtype=torch.long
68
+ ),
69
+ "prev_output_tokens": tgt.clone(),
70
+ },
71
+ "target": tgt,
72
+ "nsentences": bsz,
73
+ "ntokens": bsz * self.args.tgt_len,
74
+ },
75
+ num_items=self.args.dataset_size,
76
+ item_size=item_size,
77
+ )
78
+
79
+ @property
80
+ def source_dictionary(self):
81
+ return self.dictionary
82
+
83
+ @property
84
+ def target_dictionary(self):
85
+ return self.dictionary
86
+
87
+
88
+ class DummyDataset(FairseqDataset):
89
+ def __init__(self, batch, num_items, item_size):
90
+ super().__init__()
91
+ self.batch = batch
92
+ self.num_items = num_items
93
+ self.item_size = item_size
94
+
95
+ def __getitem__(self, index):
96
+ return index
97
+
98
+ def __len__(self):
99
+ return self.num_items
100
+
101
+ def collater(self, samples):
102
+ return self.batch
103
+
104
+ @property
105
+ def sizes(self):
106
+ return np.array([self.item_size] * self.num_items)
107
+
108
+ def num_tokens(self, index):
109
+ return self.item_size
110
+
111
+ def size(self, index):
112
+ return self.item_size
113
+
114
+ def ordered_indices(self):
115
+ return np.arange(self.num_items)
116
+
117
+ @property
118
+ def supports_prefetch(self):
119
+ return False
data/fairseq/fairseq/model_parallel/criterions/__init__.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import importlib
7
+ import os
8
+
9
+
10
+ # automatically import any Python files in the criterions/ directory
11
+ for file in sorted(os.listdir(os.path.dirname(__file__))):
12
+ if file.endswith(".py") and not file.startswith("_"):
13
+ module = file[: file.find(".py")]
14
+ importlib.import_module("fairseq.model_parallel.criterions." + module)
data/fairseq/fairseq/model_parallel/criterions/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (446 Bytes). View file
 
data/fairseq/fairseq/model_parallel/criterions/__pycache__/vocab_parallel_cross_entropy.cpython-38.pyc ADDED
Binary file (3.6 kB). View file
 
data/fairseq/fairseq/model_parallel/criterions/vocab_parallel_cross_entropy.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+
8
+ from fairseq import utils
9
+ from fairseq.logging import metrics
10
+ from fairseq.criterions import FairseqCriterion, register_criterion
11
+
12
+
13
+ try:
14
+ from fairseq.model_parallel.megatron.mpu.cross_entropy import (
15
+ vocab_parallel_cross_entropy,
16
+ )
17
+
18
+ has_megatron_submodule = True
19
+ except (ImportError, ModuleNotFoundError):
20
+ has_megatron_submodule = False
21
+
22
+
23
+ @register_criterion("vocab_parallel_cross_entropy")
24
+ class VocabParallelCrossEntropyCriterion(FairseqCriterion):
25
+ def __init__(self, task, sentence_avg):
26
+ super().__init__(task)
27
+ self.sentence_avg = sentence_avg
28
+ if not has_megatron_submodule:
29
+ raise ImportError(
30
+ "\n\nPlease install the megatron submodule:"
31
+ "\n\n git submodule update --init "
32
+ "fairseq/model_parallel/megatron"
33
+ )
34
+
35
+ def forward(self, model, sample, reduce=True):
36
+ """Compute the loss for the given sample.
37
+
38
+ Returns a tuple with three elements:
39
+ 1) the loss
40
+ 2) the sample size, which is used as the denominator for the gradient
41
+ 3) logging outputs to display while training
42
+ """
43
+ net_output = model(**sample["net_input"])
44
+ target = sample["target"]
45
+
46
+ loss = vocab_parallel_cross_entropy(net_output[0].float(), target)
47
+ loss = (loss * (target != self.padding_idx)).sum()
48
+ sample_size = (
49
+ sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
50
+ )
51
+ logging_output = {
52
+ "loss": utils.item(loss.data) if reduce else loss.data,
53
+ "ntokens": sample["ntokens"],
54
+ "nsentences": sample["target"].size(0),
55
+ "sample_size": sample_size,
56
+ }
57
+ return loss, sample_size, logging_output
58
+
59
+ @staticmethod
60
+ def reduce_metrics(logging_outputs) -> None:
61
+ """Aggregate logging outputs from data parallel training."""
62
+ loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
63
+ ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
64
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
65
+
66
+ metrics.log_scalar(
67
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
68
+ )
69
+ if sample_size != ntokens:
70
+ metrics.log_scalar(
71
+ "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
72
+ )
73
+ metrics.log_derived(
74
+ "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
75
+ )
76
+ else:
77
+ metrics.log_derived(
78
+ "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
79
+ )
80
+
81
+ @staticmethod
82
+ def logging_outputs_can_be_summed() -> bool:
83
+ """
84
+ Whether the logging outputs returned by `forward` can be summed
85
+ across workers prior to calling `reduce_metrics`. Setting this
86
+ to True will improves distributed training speed.
87
+ """
88
+ return True
data/fairseq/fairseq/model_parallel/modules/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (451 Bytes). View file
 
data/fairseq/fairseq/model_parallel/modules/__pycache__/multihead_attention.cpython-38.pyc ADDED
Binary file (7.16 kB). View file
 
data/fairseq/fairseq/model_parallel/modules/__pycache__/transformer_layer.cpython-38.pyc ADDED
Binary file (2.9 kB). View file