craftslab commited on
Commit
1fe409f
·
verified ·
1 Parent(s): d6a35bc

Upload AFM-D Encoder (Laya-layout) checkpoint

Browse files
encoder/config.json ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "ModernBertForMaskedLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 50281,
8
+ "classifier_activation": "gelu",
9
+ "classifier_bias": false,
10
+ "classifier_dropout": 0.0,
11
+ "classifier_pooling": "mean",
12
+ "cls_token_id": 50281,
13
+ "decoder_bias": true,
14
+ "deterministic_flash_attn": false,
15
+ "dtype": "float32",
16
+ "embedding_dropout": 0.0,
17
+ "eos_token_id": 50282,
18
+ "global_attn_every_n_layers": 3,
19
+ "gradient_checkpointing": false,
20
+ "hidden_activation": "gelu",
21
+ "hidden_size": 1024,
22
+ "initializer_cutoff_factor": 2.0,
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 2624,
25
+ "layer_norm_eps": 1e-05,
26
+ "layer_types": [
27
+ "full_attention",
28
+ "sliding_attention",
29
+ "sliding_attention",
30
+ "full_attention",
31
+ "sliding_attention",
32
+ "sliding_attention",
33
+ "full_attention",
34
+ "sliding_attention",
35
+ "sliding_attention",
36
+ "full_attention",
37
+ "sliding_attention",
38
+ "sliding_attention",
39
+ "full_attention",
40
+ "sliding_attention",
41
+ "sliding_attention",
42
+ "full_attention",
43
+ "sliding_attention",
44
+ "sliding_attention",
45
+ "full_attention",
46
+ "sliding_attention",
47
+ "sliding_attention",
48
+ "full_attention",
49
+ "sliding_attention",
50
+ "sliding_attention",
51
+ "full_attention",
52
+ "sliding_attention",
53
+ "sliding_attention",
54
+ "full_attention"
55
+ ],
56
+ "local_attention": 128,
57
+ "max_position_embeddings": 8192,
58
+ "mlp_bias": false,
59
+ "mlp_dropout": 0.0,
60
+ "model_type": "modernbert",
61
+ "norm_bias": false,
62
+ "norm_eps": 1e-05,
63
+ "num_attention_heads": 16,
64
+ "num_hidden_layers": 28,
65
+ "pad_token_id": 50283,
66
+ "position_embedding_type": "absolute",
67
+ "repad_logits_with_grad": false,
68
+ "rope_parameters": {
69
+ "full_attention": {
70
+ "rope_theta": 160000.0,
71
+ "rope_type": "default"
72
+ },
73
+ "sliding_attention": {
74
+ "rope_theta": 10000.0,
75
+ "rope_type": "default"
76
+ }
77
+ },
78
+ "sep_token_id": 50282,
79
+ "sparse_pred_ignore_index": -100,
80
+ "sparse_prediction": false,
81
+ "tie_word_embeddings": true,
82
+ "transformers_version": "5.17.0",
83
+ "vocab_size": 50368
84
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24c547bb5497396ff235070e7370e0dd254362393efbbab0f5d29f3e60afda06
3
+ size 842609220
rl_agent_config.json ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "encoder": "answerdotai/ModernBERT-large",
3
+ "head_layers": 2,
4
+ "max_len": 1024,
5
+ "head_max_len": 512,
6
+ "amp_dtype": "bf16",
7
+ "act_costs": {
8
+ "escalate": 0.5
9
+ },
10
+ "temperature": [
11
+ 1.3,
12
+ 1.15,
13
+ 1.4
14
+ ],
15
+ "temperature_by_options": {
16
+ "choice:11+": 1.2,
17
+ "choice:2": 1.3,
18
+ "choice:3-5": 1.35,
19
+ "choice:6-10": 1.0,
20
+ "noul:2": 1.4,
21
+ "score:3-5": 1.15
22
+ },
23
+ "confidence_temperature_by_options": {
24
+ "choice:11+": 1.2,
25
+ "choice:2": 1.3,
26
+ "choice:3-5": 1.35,
27
+ "choice:6-10": 1.0,
28
+ "noul:2": 1.4,
29
+ "score:3-5": 1.15
30
+ },
31
+ "decision_temperature_by_options": {},
32
+ "noul_confidence_mode": "max",
33
+ "choice_confidence_mode": "max",
34
+ "score_confidence_mode": "max",
35
+ "prompt_sha256": "084f1996dcf1dd2f4954803e4d7bf58c71b70b36178de3532dceab01293f1b43",
36
+ "prompt_version": "afm-d-encoder-v1",
37
+ "from_scratch": false,
38
+ "base_repo": "convaiinnovations/laya",
39
+ "training": {
40
+ "updates": 7313,
41
+ "epochs_completed": 1,
42
+ "hours": 1.96,
43
+ "world_size": 1,
44
+ "fine_tuned_from_checkpoint": true
45
+ },
46
+ "max_prefixes": 6,
47
+ "cost_wrong_act": 3.0,
48
+ "model_name": "rl-agent",
49
+ "typed_holdout_best_epoch": 1,
50
+ "typed_holdout_best_top1": 0.38738738738738737,
51
+ "select_best_score": 0.38738738738738737,
52
+ "typed_holdout_history": [
53
+ {
54
+ "epoch": 1,
55
+ "accuracy": 0.69,
56
+ "hard_accuracy": 0.38738738738738737,
57
+ "select_score": 0.38738738738738737,
58
+ "soft_cross_entropy": 0.9388098226946757,
59
+ "beats_agentjev_top1": false,
60
+ "reference_jevbench_public_gate": 50.0
61
+ },
62
+ {
63
+ "epoch": 2,
64
+ "accuracy": 0.7233333333333334,
65
+ "hard_accuracy": 0.32432432432432434,
66
+ "select_score": 0.32432432432432434,
67
+ "soft_cross_entropy": 0.8705833858013335,
68
+ "beats_agentjev_top1": false,
69
+ "reference_jevbench_public_gate": 50.0
70
+ },
71
+ {
72
+ "epoch": 3,
73
+ "accuracy": 0.7733333333333333,
74
+ "hard_accuracy": 0.32432432432432434,
75
+ "select_score": 0.32432432432432434,
76
+ "soft_cross_entropy": 0.8696609151637089,
77
+ "beats_agentjev_top1": false,
78
+ "reference_jevbench_public_gate": 50.0
79
+ },
80
+ {
81
+ "epoch": 4,
82
+ "accuracy": 0.7683333333333333,
83
+ "hard_accuracy": 0.3153153153153153,
84
+ "select_score": 0.3153153153153153,
85
+ "soft_cross_entropy": 0.8556363284158692,
86
+ "beats_agentjev_top1": false,
87
+ "reference_jevbench_public_gate": 50.0
88
+ }
89
+ ],
90
+ "confidence_modes": {
91
+ "noul": "max",
92
+ "choice": "max",
93
+ "score": "max"
94
+ }
95
+ }
tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "clean_up_tokenization_spaces": true,
3
+ "cls_token": "[CLS]",
4
+ "local_files_only": false,
5
+ "mask_token": "[MASK]",
6
+ "model_input_names": [
7
+ "input_ids",
8
+ "attention_mask"
9
+ ],
10
+ "model_max_length": 8192,
11
+ "pad_token": "[PAD]",
12
+ "sep_token": "[SEP]",
13
+ "tokenizer_class": "PreTrainedTokenizerFast",
14
+ "unk_token": "[UNK]"
15
+ }