mahwizzzz commited on
Commit
cebac2b
·
verified ·
1 Parent(s): 24c100d

Add Transformers AutoModelForMaskedLM remote code

Browse files
Files changed (1) hide show
  1. config.json +33 -75
config.json CHANGED
@@ -1,75 +1,33 @@
1
- {
2
- "name": "pakmosaic_small",
3
- "hidden_size": 512,
4
- "num_hidden_layers": 12,
5
- "num_attention_heads": 8,
6
- "intermediate_size": 2048,
7
- "vocab_size": 32000,
8
- "max_position_embeddings": 512,
9
- "rope_theta": 10000.0,
10
- "attention_dropout": 0.0,
11
- "hidden_dropout": 0.0,
12
- "ffn_activation": "geglu",
13
- "attention_kind": "full",
14
- "local_window": 128,
15
- "global_every": 3,
16
- "use_bias": false,
17
- "tie_word_embeddings": true,
18
- "pad_token_id": 0,
19
- "bos_token_id": 1,
20
- "eos_token_id": 2,
21
- "unk_token_id": 3,
22
- "mask_token_id": 4,
23
- "layer_norm_eps": 1e-05,
24
- "initializer_range": 0.02,
25
- "tokenizer_id": "hf_byte_bpe_32k_identity",
26
- "public_name": "PakMosaic-Small",
27
- "script_signal": "none",
28
- "sampling": {
29
- "kind": "temperature",
30
- "temperature": 3.0,
31
- "anneal_start_temperature": 5.0,
32
- "anneal_end_temperature": 1.0,
33
- "min_language_prob": 0.01,
34
- "languages": [
35
- "pus_Arab",
36
- "urd_Arab",
37
- "pnb_Arab",
38
- "skr_Arab",
39
- "snd_Arab",
40
- "kas_Arab",
41
- "guj_Gujr"
42
- ]
43
- },
44
- "masking": {
45
- "kind": "fixed",
46
- "mask_prob": 0.15,
47
- "scheduled_peak": 0.3,
48
- "replace_prob": 0.8,
49
- "random_prob": 0.1,
50
- "keep_prob": 0.1
51
- },
52
- "train": {
53
- "max_seq_len": 512,
54
- "smoke_seq_len": 128,
55
- "token_budget": 2000000000,
56
- "micro_batch_size": 2,
57
- "grad_accum": 8,
58
- "learning_rate": 0.0003,
59
- "weight_decay": 0.01,
60
- "warmup_steps": 50,
61
- "max_grad_norm": 1.0,
62
- "seed": 42,
63
- "log_every": 10,
64
- "checkpoint_every": 20000,
65
- "keep_last_checkpoints": 2,
66
- "require_expensive_confirm": true
67
- },
68
- "algebraic_parameters": {
69
- "embeddings": 16384000,
70
- "layers": 50343936,
71
- "mlm_head": 263168,
72
- "total": 66991616,
73
- "trainable": 66991616
74
- }
75
- }
 
1
+ {
2
+ "architectures": ["PakMosaicForMaskedLM"],
3
+ "model_type": "pakmosaic",
4
+ "auto_map": {
5
+ "AutoConfig": "configuration_pakmosaic.PakMosaicConfig",
6
+ "AutoModel": "modeling_pakmosaic.PakMosaicModel",
7
+ "AutoModelForMaskedLM": "modeling_pakmosaic.PakMosaicForMaskedLM"
8
+ },
9
+ "vocab_size": 32000,
10
+ "hidden_size": 512,
11
+ "num_hidden_layers": 12,
12
+ "num_attention_heads": 8,
13
+ "intermediate_size": 2048,
14
+ "max_position_embeddings": 512,
15
+ "rope_theta": 10000.0,
16
+ "attention_dropout": 0.0,
17
+ "hidden_dropout": 0.0,
18
+ "ffn_activation": "geglu",
19
+ "attention_kind": "full",
20
+ "local_window": 128,
21
+ "global_every": 3,
22
+ "use_bias": false,
23
+ "tie_word_embeddings": true,
24
+ "pad_token_id": 0,
25
+ "bos_token_id": 1,
26
+ "eos_token_id": 2,
27
+ "unk_token_id": 3,
28
+ "mask_token_id": 4,
29
+ "layer_norm_eps": 1e-05,
30
+ "initializer_range": 0.02,
31
+ "script_signal": "none",
32
+ "tokenizer_class": "PreTrainedTokenizerFast"
33
+ }