dhtocks commited on
Commit
3767058
·
verified ·
1 Parent(s): 8a9eb21

Upload k4b-mix4-step4500 (base Qwen/Qwen3.5-4B-Base; acc nan, ECE nan)

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: Qwen/Qwen3.5-4B-Base
3
+ base_model_relation: adapter
4
+ library_name: peft
5
+ license: cc-by-nc-4.0
6
+ language: [en, ko, ja, zh, de, fr, es, hi, ar, th]
7
+ tags: [decision-model, classification, calibration, kev, system-one, lora, pointer-head]
8
+ ---
9
+
10
+ # Malkuth-4B
11
+
12
+ A small decision model for multilingual and Korean classification. Given text, a question and a set of criteria, it
13
+ returns a probability distribution over the options instead of generating an answer. Post-trained from
14
+ [Kev](https://github.com/jaredpalmer/kev) and compatible with TypeSafe's System One API.
15
+
16
+ Code, training mixes and evaluation: [newfull5/malkuth](https://github.com/newfull5/malkuth).
17
+ Companion model: [dhtocks/malkuth-2b](https://huggingface.co/dhtocks/malkuth-2b) (2B).
18
+
19
+ ## Model
20
+
21
+ - Rank-16 LoRA adapter and a pointer head on a frozen `Qwen/Qwen3.5-4B-Base`; 33.8M trainable parameters, 124 MB adapter
22
+ - `choice` (1-255 options), `noul` (yes/no) and `score` (ordered levels), several questions per request
23
+ - Questions share the input text but cannot read each other, so one document can be classified, checked and scored at once
24
+ - Probabilities are temperature-calibrated on a validation split; the temperature never changes which option wins
25
+
26
+ ## Usage
27
+
28
+ Serve with Kev; the official `typesafe-sdk` works against it with a `base_url` change.
29
+
30
+ ```bash
31
+ git clone https://github.com/jaredpalmer/kev.git kev && cd kev && uv sync --extra serve
32
+ uv run --extra serve python -m kev.serve --run dhtocks/malkuth-4b --port 8009
33
+ ```
34
+
35
+ ```python
36
+ from typesafe_sdk import Choice, Noul, Score, TypeSafeClient
37
+
38
+ client = TypeSafeClient(api_key="local", base_url="http://127.0.0.1:8009", model="kev-latest")
39
+ r = client.system_one(
40
+ state="주문한 신발이 2주 늦게, 다른 사이즈로 왔습니다. 카드에 결제도 두 번 찍혔어요.",
41
+ questions={
42
+ "team": Choice(instructions="어느 팀이 처리해야 합니까?",
43
+ criteria={"billing": "결제·청구", "shipping": "배송", "returns": "교환·환불"}),
44
+ "escalate": Noul(instructions="사람이 즉시 개입해야 합니까?"),
45
+ "urgency": Score(instructions="얼마나 급합니까?", criteria=["낮음", "보통", "높음"]),
46
+ },
47
+ )
48
+ print(r.choices["team"].choice, r.choices["team"].probabilities)
49
+ ```
50
+
51
+ ## Evaluation
52
+
53
+ 29 suites, about 129,000 questions, the same items for every model; Jev measured through its API.
54
+ **held-out** is the nine datasets absent from the training mix — the number to read. **all** includes ten we trained
55
+ on. Brier scores the probabilities, not just the chosen answer; lower is better.
56
+
57
+ | Model | Accuracy (held-out) ↑ | Accuracy (all) ↑ | Brier (held-out) ↓ |
58
+ |---|---:|---:|---:|
59
+ | Jev API | **0.754** ±0.0042 | 0.790 | **0.355** |
60
+ | **Malkuth-4B** | 0.724 ±0.0044 | **0.794** | 0.385 |
61
+ | **Malkuth-2B** | 0.703 ±0.0044 | 0.765 | 0.412 |
62
+ | Kev-9B | 0.700 ±0.0046 | 0.737 | 0.407 |
63
+ | Kev-4B | 0.686 ±0.0046 | 0.720 | 0.407 |
64
+ | Kev-0.8B | 0.589 ±0.0050 | 0.584 | 0.539 |
65
+ | Laya (best of four) | 0.544 ±0.0044 | 0.507 | 0.654 |
66
+
67
+ Model time per request on one A100 80GB in bf16, median of 20, new state / prefix-cached state: 48 / 32 ms for six questions, 145 / 33 ms on a 2,200-token state.
68
+
69
+ Per-dataset scores, calibration analysis and third-party test sets:
70
+ [docs/evaluation.md](https://github.com/newfull5/malkuth/blob/main/docs/evaluation.md).
71
+
72
+ ## Training data
73
+
74
+ 89,791 labelled requests from 20 public sources — 62,400 `choice`, 24,790 `noul` and 7,401 `score` questions, across
75
+ ten core languages with five Korean-only sources. Built by
76
+ [`tools/build_train_mix.py`](https://github.com/newfull5/malkuth/blob/main/tools/build_train_mix.py); per-source
77
+ licences in [docs/benchmarks.md](https://github.com/newfull5/malkuth/blob/main/docs/benchmarks.md).
78
+
79
+ ## License
80
+
81
+ - **Weights** - the training mixes include CC-BY-NC datasets.
82
+ - **Code** - Apache-2.0, same as Kev and the Qwen bases.
83
+
84
+ ## Citation
85
+
86
+ ```bibtex
87
+ @misc{malkuth2026,
88
+ title={Malkuth: multilingual System One decision models},
89
+ author={Saechan Oh},
90
+ year={2026}
91
+ }
92
+ ```
adapter_config.json ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "kasa_config": null,
16
+ "layer_replication": null,
17
+ "layers_pattern": null,
18
+ "layers_to_transform": null,
19
+ "loftq_config": {},
20
+ "lora_alpha": 32,
21
+ "lora_bias": false,
22
+ "lora_dropout": 0.05,
23
+ "lora_ga_config": null,
24
+ "megatron_config": null,
25
+ "megatron_core": "megatron.core",
26
+ "modules_to_save": null,
27
+ "monteclora_config": null,
28
+ "peft_type": "LORA",
29
+ "peft_version": "0.21.0",
30
+ "qalora_group_size": 16,
31
+ "r": 16,
32
+ "rank_pattern": {},
33
+ "revision": null,
34
+ "target_modules": [
35
+ "down_proj",
36
+ "o_proj",
37
+ "in_proj_qkv",
38
+ "in_proj_b",
39
+ "k_proj",
40
+ "in_proj_a",
41
+ "in_proj_z",
42
+ "up_proj",
43
+ "q_proj",
44
+ "v_proj",
45
+ "gate_proj",
46
+ "out_proj"
47
+ ],
48
+ "target_parameters": null,
49
+ "task_type": "FEATURE_EXTRACTION",
50
+ "trainable_token_indices": null,
51
+ "use_bdlora": null,
52
+ "use_dora": false,
53
+ "use_qalora": false,
54
+ "use_rslora": false,
55
+ "velora_config": null
56
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96b4f5a922afe581a5dd6bca23e8126aaca447dd59f634935feea77a326ed1c1
3
+ size 129924032
head.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e3e63bf28a0447f8b12f0479afc763d33c7d975f77bb1808a4857aca1bc80d3
3
+ size 5248767
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
3
+ size 19989325
tokenizer_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "audio_bos_token": "<|audio_start|>",
4
+ "audio_eos_token": "<|audio_end|>",
5
+ "audio_token": "<|audio_pad|>",
6
+ "backend": "tokenizers",
7
+ "bos_token": null,
8
+ "clean_up_tokenization_spaces": false,
9
+ "eos_token": "<|endoftext|>",
10
+ "errors": "replace",
11
+ "image_token": "<|image_pad|>",
12
+ "is_local": false,
13
+ "local_files_only": false,
14
+ "model_max_length": 262144,
15
+ "model_specific_special_tokens": {
16
+ "audio_bos_token": "<|audio_start|>",
17
+ "audio_eos_token": "<|audio_end|>",
18
+ "audio_token": "<|audio_pad|>",
19
+ "image_token": "<|image_pad|>",
20
+ "video_token": "<|video_pad|>",
21
+ "vision_bos_token": "<|vision_start|>",
22
+ "vision_eos_token": "<|vision_end|>"
23
+ },
24
+ "pad_token": "<|endoftext|>",
25
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
26
+ "split_special_tokens": false,
27
+ "tokenizer_class": "Qwen2Tokenizer",
28
+ "unk_token": null,
29
+ "video_token": "<|video_pad|>",
30
+ "vision_bos_token": "<|vision_start|>",
31
+ "vision_eos_token": "<|vision_end|>"
32
+ }