Add files using upload-large-folder tool

CALIBRATION.json

@@ -0,0 +1,136 @@
+{
+ "prompt": 2,
+ "orders": 1,
+ "method": "temperature per question type and per option-count bucket (choice), by NLL, fitted on the out-of-distribution calibration split only; thresholds per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
+ "temperature_per_prim": {
+  "choice": 1.3939382634489992,
+  "noul": 2.82842712474619,
+  "score": 2.44810708660931
+ },
+ "temperature_per_bucket": {
+  "choice": {
+   "2-10": 1.8340080864093422,
+   "11-26": 1.241857812073484,
+   "27-255": 1.059463094359295
+  }
+ },
+ "thresholds": {
+  "0.01": {
+   "confidence": null,
+   "method": "per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
+   "per_type": {
+    "choice": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    },
+    "noul": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    },
+    "score": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    }
+   },
+   "heldout_coverage": 0.0,
+   "heldout_error": null
+  },
+  "0.05": {
+   "confidence": null,
+   "method": "per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
+   "per_type": {
+    "choice": {
+     "confidence": 0.9322699589043795,
+     "heldout_coverage": 0.3940542361462018,
+     "heldout_error": 0.023936738619363074
+    },
+    "noul": {
+     "confidence": 0.8915736513972552,
+     "heldout_coverage": 0.37426679280983915,
+     "heldout_error": 0.020222446916076886
+    },
+    "score": {
+     "confidence": null,
+     "heldout_coverage": 0.01175285426460712,
+     "heldout_error": 0.22857142857142854
+    }
+   },
+   "heldout_coverage": 0.33232358345334456,
+   "heldout_error": 0.02390914524805743
+  }
+ },
+ "gates": {
+  "ece_lofo": {
+   "choice": {
+    "pooled": 0.012308234464218113,
+    "n": 11874,
+    "gate": 0.05,
+    "pass": true
+   },
+   "noul": {
+    "pooled": 0.03186722948382228,
+    "n": 5285,
+    "gate": 0.05,
+    "pass": true
+   },
+   "score": {
+    "pooled": 0.030276060779249152,
+    "n": 2978,
+    "gate": 0.05,
+    "pass": true
+   }
+  },
+  "ece_worst_fold": 0.1014424717632049,
+  "worst_fold": "agent",
+  "automated_at": {
+   "0.01": 0.0,
+   "0.05": 0.33232358345334456
+  },
+  "automated_error": {
+   "0.01": null,
+   "0.05": 0.02390914524805743
+  },
+  "note": "held-out numbers: every evaluation row predicted by the fit without its family"
+ },
+ "test": {
+  "T": {
+   "choice": {
+    "acc": 0.7870136432541688,
+    "ece": 0.014067322629837065,
+    "n": 11874
+   },
+   "noul": {
+    "acc": 0.8306527909176916,
+    "ece": 0.03343366640192696,
+    "n": 5285
+   },
+   "score": {
+    "acc": 0.6040967092008059,
+    "ece": 0.02988934980023047,
+    "n": 2978
+   },
+   "all": {
+    "acc": 0.771415801757958,
+    "ece": 0.01582616023538169,
+    "n": 20137
+   }
+  },
+  "note": "evaluation pool (development and test splits), full fit"
+ },
+ "n": {
+  "fit": 3560,
+  "many": 1500,
+  "eval": 20137
+ },
+ "engine": "fast",
+ "fitted_on": {
+  "split": "out-of-distribution calibration split (sealed-seed BH-, JDI- and agent-shaped families, many-option items)",
+  "orders": 1,
+  "readouts": "basal-serve --mode fast, bf16"
+ },
+ "certified_on": "the same split",
+ "evaluated_on": "development and test splits, leave-one-family-out and leave-one-language-out"
+}

CALIBRATION.o2.json

@@ -0,0 +1,136 @@
+{
+ "prompt": 2,
+ "orders": 2,
+ "method": "temperature per question type and per option-count bucket (choice), by NLL, fitted on the out-of-distribution calibration split only; thresholds per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
+ "temperature_per_prim": {
+  "choice": 1.2968395546510096,
+  "noul": 2.82842712474619,
+  "score": 2.2775772695133827
+ },
+ "temperature_per_bucket": {
+  "choice": {
+   "2-10": 1.731073122012286,
+   "11-26": 1.1224620483093728,
+   "27-255": 0.9576032806985735
+  }
+ },
+ "thresholds": {
+  "0.01": {
+   "confidence": null,
+   "method": "per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
+   "per_type": {
+    "choice": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    },
+    "noul": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    },
+    "score": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    }
+   },
+   "heldout_coverage": 0.0,
+   "heldout_error": null
+  },
+  "0.05": {
+   "confidence": null,
+   "method": "per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
+   "per_type": {
+    "choice": {
+     "confidence": 0.9445740561766649,
+     "heldout_coverage": 0.3873168266801415,
+     "heldout_error": 0.023700804522722385
+    },
+    "noul": {
+     "confidence": 0.9555549119776258,
+     "heldout_coverage": 0.13509933774834437,
+     "heldout_error": 0.009803921568627416
+    },
+    "score": {
+     "confidence": 0.8910746290731381,
+     "heldout_coverage": 0.1618535930154466,
+     "heldout_error": 0.024896265560165998
+    }
+   },
+   "heldout_coverage": 0.287778715796792,
+   "heldout_error": 0.022088006902502122
+  }
+ },
+ "gates": {
+  "ece_lofo": {
+   "choice": {
+    "pooled": 0.022320540236599275,
+    "n": 11874,
+    "gate": 0.05,
+    "pass": true
+   },
+   "noul": {
+    "pooled": 0.0344665822105491,
+    "n": 5285,
+    "gate": 0.05,
+    "pass": true
+   },
+   "score": {
+    "pooled": 0.030798399068276784,
+    "n": 2978,
+    "gate": 0.05,
+    "pass": true
+   }
+  },
+  "ece_worst_fold": 0.09938740135848097,
+  "worst_fold": "agent",
+  "automated_at": {
+   "0.01": 0.0,
+   "0.05": 0.287778715796792
+  },
+  "automated_error": {
+   "0.01": null,
+   "0.05": 0.022088006902502122
+  },
+  "note": "held-out numbers: every evaluation row predicted by the fit without its family"
+ },
+ "test": {
+  "T": {
+   "choice": {
+    "acc": 0.7947616641401382,
+    "ece": 0.021102276315294953,
+    "n": 11874
+   },
+   "noul": {
+    "acc": 0.8302743614001892,
+    "ece": 0.034630485034101086,
+    "n": 5285
+   },
+   "score": {
+    "acc": 0.6124916051040967,
+    "ece": 0.030901334113014996,
+    "n": 2978
+   },
+   "all": {
+    "acc": 0.7771266822267467,
+    "ece": 0.01930530754099537,
+    "n": 20137
+   }
+  },
+  "note": "evaluation pool (development and test splits), full fit"
+ },
+ "n": {
+  "fit": 3560,
+  "many": 1500,
+  "eval": 20137
+ },
+ "engine": "fast",
+ "fitted_on": {
+  "split": "out-of-distribution calibration split (sealed-seed BH-, JDI- and agent-shaped families, many-option items)",
+  "orders": 2,
+  "readouts": "basal-serve --mode fast, bf16"
+ },
+ "certified_on": "the same split",
+ "evaluated_on": "development and test splits, leave-one-family-out and leave-one-language-out"
+}

CALIBRATION.think.json

@@ -0,0 +1,135 @@
+{
+ "prompt": 2,
+ "orders": 1,
+ "method": "temperature per question type and per option-count bucket (choice), by NLL, fitted on the out-of-distribution calibration split only; thresholds per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
+ "temperature_per_prim": {
+  "choice": 2.996614153753363,
+  "noul": 7.999999999999998,
+  "score": 4.756828460010884
+ },
+ "temperature_per_bucket": {
+  "choice": {
+   "2-10": 3.830413122794294,
+   "11-26": 2.747907294916178,
+   "27-255": 1.731073122012286
+  }
+ },
+ "thresholds": {
+  "0.01": {
+   "confidence": null,
+   "method": "per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
+   "per_type": {
+    "choice": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    },
+    "noul": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    },
+    "score": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    }
+   },
+   "heldout_coverage": 0.0,
+   "heldout_error": null
+  },
+  "0.05": {
+   "confidence": null,
+   "method": "per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
+   "per_type": {
+    "choice": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    },
+    "noul": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    },
+    "score": {
+     "confidence": null,
+     "heldout_coverage": 0.0,
+     "heldout_error": null
+    }
+   },
+   "heldout_coverage": 0.0,
+   "heldout_error": null
+  }
+ },
+ "gates": {
+  "ece_lofo": {
+   "choice": {
+    "pooled": 0.0606221052989346,
+    "n": 7229,
+    "gate": 0.05,
+    "pass": false
+   },
+   "noul": {
+    "pooled": 0.05231366996904953,
+    "n": 3197,
+    "gate": 0.05,
+    "pass": false
+   },
+   "score": {
+    "pooled": 0.0740364665334774,
+    "n": 2978,
+    "gate": 0.05,
+    "pass": false
+   }
+  },
+  "ece_worst_fold": 0.12149008840247141,
+  "worst_fold": "agent",
+  "automated_at": {
+   "0.01": 0.0,
+   "0.05": 0.0
+  },
+  "automated_error": {
+   "0.01": null,
+   "0.05": null
+  },
+  "note": "held-out numbers: every evaluation row predicted by the fit without its family"
+ },
+ "test": {
+  "T": {
+   "choice": {
+    "acc": 0.6067229215659151,
+    "ece": 0.05329490346712401,
+    "n": 7229
+   },
+   "noul": {
+    "acc": 0.737566468564279,
+    "ece": 0.05231366996904953,
+    "n": 3197
+   },
+   "score": {
+    "acc": 0.5957018132975151,
+    "ece": 0.08010245549854482,
+    "n": 2978
+   },
+   "all": {
+    "acc": 0.6354819456878543,
+    "ece": 0.0323180305717361,
+    "n": 13404
+   }
+  },
+  "note": "evaluation pool (development and test splits), full fit"
+ },
+ "n": {
+  "fit": 2614,
+  "many": 885,
+  "eval": 13404
+ },
+ "engine": "think",
+ "fitted_on": {
+  "split": "out-of-distribution calibration split (sealed-seed BH-, JDI- and agent-shaped families, many-option items)",
+  "orders": 1
+ },
+ "certified_on": "the same split",
+ "evaluated_on": "development and test splits, leave-one-family-out and leave-one-language-out"
+}

SHA256SUMS

@@ -0,0 +1,17 @@
+84a24109362342048505093fd06c8ccf213d5e10d5835542f6d599e297388d49  CALIBRATION.json
+ede2d44cfdf9736b93e2e0d6bc7b68f27deff633bc3268361cb28cfefaecc2ab  CALIBRATION.o2.json
+8938f99af2d23223e197cfbd52e3941627934cf33dd132fd310d1a84db9b217d  CALIBRATION.think.json
+7604559595d509bf29defa40e8394a0b611d05ce3cd0fca0938a3ee6627a1493  basal.json
+a805e50fed68938a076b07e2e602639611b50b1ced0e50f11eb92f1ba25be4dc  chat_template.jinja
+2ac96f2e6b9731382bee55320062498d06272542b7fd2faa4bef04065ba1e2c3  config.json
+76594a7e29cc06ab1f219d175a8ee72e6a470053040a5156fb22226c1c6cfe9c  generation_config.json
+481d8ea4972379a611f46784196e407bee239a37c3981c6e29ff1aaaeb0f19c2  model.safetensors
+7604559595d509bf29defa40e8394a0b611d05ce3cd0fca0938a3ee6627a1493  think/basal.json
+a805e50fed68938a076b07e2e602639611b50b1ced0e50f11eb92f1ba25be4dc  think/chat_template.jinja
+2ac96f2e6b9731382bee55320062498d06272542b7fd2faa4bef04065ba1e2c3  think/config.json
+76594a7e29cc06ab1f219d175a8ee72e6a470053040a5156fb22226c1c6cfe9c  think/generation_config.json
+1e295ab2bcc6e812bc9e22f710cd0a6fc4d91371eefffcb271d77cb6ad97c8a5  think/model.safetensors
+20f059c5ba108cc6e4d59df55be30e73c30b86ff7aab314362bfa4eff1e21657  think/tokenizer.json
+b90dd402080d2558a60b82033d71e3c1ef750e0811709ae166c0f7d3fa0693cd  think/tokenizer_config.json
+20f059c5ba108cc6e4d59df55be30e73c30b86ff7aab314362bfa4eff1e21657  tokenizer.json
+b90dd402080d2558a60b82033d71e3c1ef750e0811709ae166c0f7d3fa0693cd  tokenizer_config.json

basal.json

@@ -0,0 +1,15 @@
+{
+ "name": "basal-2.0-4.5B",
+ "prompt": 2,
+ "readout": {
+  "kind": "code logits",
+  "codes": 255,
+  "prefill": "{\"answer\": \"",
+  "templates": "prompt format 2 (docs/FORMAT2.md of the basal engine)"
+ },
+ "calibration": "CALIBRATION.json (per-type and per-bucket temperatures, certified thresholds)",
+ "test_metrics": {
+  "acc": 0.8781542056074766,
+  "n_items": 8560
+ }
+}

chat_template.jinja

@@ -0,0 +1,4 @@
+{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

config.json

@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": true,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": [
+    4,
+    2
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 32768,
+  "mlp_bias": true,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 60,
+  "num_key_value_heads": 2,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.17.0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    4,
+    2
+  ],
+  "pad_token_id": 2,
+  "transformers_version": "5.17.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:481d8ea4972379a611f46784196e407bee239a37c3981c6e29ff1aaaeb0f19c2
+size 9514629496

think/basal.json

@@ -0,0 +1,15 @@
+{
+ "name": "basal-2.0-4.5B",
+ "prompt": 2,
+ "readout": {
+  "kind": "code logits",
+  "codes": 255,
+  "prefill": "{\"answer\": \"",
+  "templates": "prompt format 2 (docs/FORMAT2.md of the basal engine)"
+ },
+ "calibration": "CALIBRATION.json (per-type and per-bucket temperatures, certified thresholds)",
+ "test_metrics": {
+  "acc": 0.8781542056074766,
+  "n_items": 8560
+ }
+}

think/chat_template.jinja

@@ -0,0 +1,4 @@
+{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

think/config.json

@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": true,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": [
+    4,
+    2
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 32768,
+  "mlp_bias": true,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 60,
+  "num_key_value_heads": 2,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.17.0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

think/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    4,
+    2
+  ],
+  "pad_token_id": 2,
+  "transformers_version": "5.17.0"
+}

think/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e295ab2bcc6e812bc9e22f710cd0a6fc4d91371eefffcb271d77cb6ad97c8a5
+size 9514629496

think/tokenizer_config.json

@@ -0,0 +1,27 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<tool_call>",
+    "</tool_call>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "max_length": null,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
+  "pad_token": "</s>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "truncation_side": "left",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

tokenizer_config.json

@@ -0,0 +1,27 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<tool_call>",
+    "</tool_call>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "max_length": null,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
+  "pad_token": "</s>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "truncation_side": "left",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}