gloomcheng commited on
Commit
12da14e
·
verified ·
1 Parent(s): 461af5a

release: update IlhaEmbed with expanded clinical lab & procedural contrastive tuning

Browse files
Files changed (1) hide show
  1. config.json +26 -3
config.json CHANGED
@@ -16,7 +16,6 @@
16
  "embedding_dropout": 0.0,
17
  "eos_token_id": 25464,
18
  "global_attn_every_n_layers": 3,
19
- "global_rope_theta": 150000.0,
20
  "gradient_checkpointing": false,
21
  "hidden_activation": "silu",
22
  "hidden_size": 384,
@@ -24,8 +23,21 @@
24
  "initializer_range": 0.02,
25
  "intermediate_size": 1536,
26
  "layer_norm_eps": 1e-05,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  "local_attention": 128,
28
- "local_rope_theta": 160000.0,
29
  "max_position_embeddings": 32768,
30
  "mlp_bias": false,
31
  "mlp_dropout": 0.0,
@@ -37,9 +49,20 @@
37
  "pad_token_id": 25461,
38
  "position_embedding_type": "absolute",
39
  "repad_logits_with_grad": false,
 
 
 
 
 
 
 
 
 
 
40
  "sep_token_id": 25464,
41
  "sparse_pred_ignore_index": -100,
42
  "sparse_prediction": false,
43
- "transformers_version": "4.57.1",
 
44
  "vocab_size": 25526
45
  }
 
16
  "embedding_dropout": 0.0,
17
  "eos_token_id": 25464,
18
  "global_attn_every_n_layers": 3,
 
19
  "gradient_checkpointing": false,
20
  "hidden_activation": "silu",
21
  "hidden_size": 384,
 
23
  "initializer_range": 0.02,
24
  "intermediate_size": 1536,
25
  "layer_norm_eps": 1e-05,
26
+ "layer_types": [
27
+ "full_attention",
28
+ "sliding_attention",
29
+ "sliding_attention",
30
+ "full_attention",
31
+ "sliding_attention",
32
+ "sliding_attention",
33
+ "full_attention",
34
+ "sliding_attention",
35
+ "sliding_attention",
36
+ "full_attention",
37
+ "sliding_attention",
38
+ "sliding_attention"
39
+ ],
40
  "local_attention": 128,
 
41
  "max_position_embeddings": 32768,
42
  "mlp_bias": false,
43
  "mlp_dropout": 0.0,
 
49
  "pad_token_id": 25461,
50
  "position_embedding_type": "absolute",
51
  "repad_logits_with_grad": false,
52
+ "rope_parameters": {
53
+ "full_attention": {
54
+ "rope_theta": 150000.0,
55
+ "rope_type": "default"
56
+ },
57
+ "sliding_attention": {
58
+ "rope_theta": 160000.0,
59
+ "rope_type": "default"
60
+ }
61
+ },
62
  "sep_token_id": 25464,
63
  "sparse_pred_ignore_index": -100,
64
  "sparse_prediction": false,
65
+ "tie_word_embeddings": true,
66
+ "transformers_version": "5.12.1",
67
  "vocab_size": 25526
68
  }