File size: 1,999 Bytes
ea51a3d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | from transformers import PretrainedConfig
class TransWaveConfig(PretrainedConfig):
model_type = "transwave_meyer"
def __init__(
self,
vocab_size=256,
byte_dim=96,
bytes_per_block=8,
num_scales=8,
hidden_size=768,
max_position_embeddings=32768,
block_context=4096,
num_hidden_layers=12,
num_groups=12,
meyer_bands=12,
meyer_controls_per_band=8,
meyer_kernel_blocks=512,
local_kernel_blocks=7,
intermediate_size=2816,
decoder_hidden_size=256,
trained_context_length=20480,
architecture_id="causal_transwave_blocklifting_meyer_byte_lm_g4_v2",
**kwargs,
):
# config.json may already supply this.
# Never provide it twice.
kwargs.setdefault(
"tie_word_embeddings",
True,
)
super().__init__(
**kwargs
)
self.vocab_size = vocab_size
self.byte_dim = byte_dim
self.bytes_per_block = bytes_per_block
self.num_scales = num_scales
self.hidden_size = hidden_size
self.max_position_embeddings = (
max_position_embeddings
)
self.block_context = block_context
self.num_hidden_layers = (
num_hidden_layers
)
self.num_groups = num_groups
self.meyer_bands = meyer_bands
self.meyer_controls_per_band = (
meyer_controls_per_band
)
self.meyer_kernel_blocks = (
meyer_kernel_blocks
)
self.local_kernel_blocks = (
local_kernel_blocks
)
self.intermediate_size = (
intermediate_size
)
self.decoder_hidden_size = (
decoder_hidden_size
)
self.trained_context_length = (
trained_context_length
)
self.architecture_id = (
architecture_id
)
|