Text Generation
PEFT
Safetensors
English
lora
olmo
uncertainty
calibration
abstention
retrieval-augmented-generation
Instructions to use Sentovara/Sentinel-S1-7B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use Sentovara/Sentinel-S1-7B with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("allenai/Olmo-3-7B-Instruct") model = PeftModel.from_pretrained(base_model, "Sentovara/Sentinel-S1-7B") - Notebooks
- Google Colab
- Kaggle
Download results/conversation_summary.json from Sentovara/Sentinel-S1-7B: direct link, hf CLI and curl.
- Browser
- Download file 5.78 kB
-
https://huggingface.co/Sentovara/Sentinel-S1-7B/resolve/main/results/conversation_summary.json
- Command line
-
hf download hf://Sentovara/Sentinel-S1-7B/results/conversation_summary.json
-
curl -L -o conversation_summary.json https://huggingface.co/Sentovara/Sentinel-S1-7B/resolve/main/results/conversation_summary.json
5.78 kB
| { | |
| "K1_unflagged_beats_original_by_10pts": { | |
| "sentinel_unflagged_accuracy": 0.8227848101265823, | |
| "base_accuracy": 0.43, | |
| "pass": true | |
| }, | |
| "K2_flags_half_its_wrong": { | |
| "sentinel_flag_rate_on_its_wrong": 0.9247311827956989, | |
| "pass": true | |
| }, | |
| "K3_clean_endings": { | |
| "sentinel_clean_end_rate": 1.0, | |
| "base_clean_end_rate": 0.9914285714285714, | |
| "pass": true | |
| }, | |
| "K4_no_false_flags_on_chat": { | |
| "sentinel_flag_rate_no_fact_chats": 0.06, | |
| "n": 50, | |
| "pass": true | |
| }, | |
| "question_conversations": { | |
| "n": 300, | |
| "base_accuracy": 0.43, | |
| "sentinel_overall_accuracy": 0.38, | |
| "sentinel_unflagged_accuracy": 0.8227848101265823, | |
| "sentinel_unflagged_n": 79, | |
| "sentinel_flag_rate": 0.7366666666666667, | |
| "sentinel_flag_rate_on_its_wrong": 0.9247311827956989, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "breakdown": { | |
| "A_loose_single/fact": { | |
| "n": 70, | |
| "base_accuracy": 0.2857142857142857, | |
| "sentinel_overall_accuracy": 0.1, | |
| "sentinel_unflagged_accuracy": 0.375, | |
| "sentinel_unflagged_n": 8, | |
| "sentinel_flag_rate": 0.8857142857142857, | |
| "sentinel_flag_rate_on_its_wrong": 0.9206349206349206, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "A_loose_single/math": { | |
| "n": 30, | |
| "base_accuracy": 0.8666666666666667, | |
| "sentinel_overall_accuracy": 1.0, | |
| "sentinel_unflagged_accuracy": 1.0, | |
| "sentinel_unflagged_n": 25, | |
| "sentinel_flag_rate": 0.16666666666666666, | |
| "sentinel_flag_rate_on_its_wrong": null, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "A_loose_single/all": { | |
| "n": 100, | |
| "base_accuracy": 0.46, | |
| "sentinel_overall_accuracy": 0.37, | |
| "sentinel_unflagged_accuracy": 0.8484848484848485, | |
| "sentinel_unflagged_n": 33, | |
| "sentinel_flag_rate": 0.67, | |
| "sentinel_flag_rate_on_its_wrong": 0.9206349206349206, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "B_multi_turn/fact": { | |
| "n": 70, | |
| "base_accuracy": 0.24285714285714285, | |
| "sentinel_overall_accuracy": 0.15714285714285714, | |
| "sentinel_unflagged_accuracy": 0.4, | |
| "sentinel_unflagged_n": 5, | |
| "sentinel_flag_rate": 0.9285714285714286, | |
| "sentinel_flag_rate_on_its_wrong": 0.9491525423728814, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "B_multi_turn/math": { | |
| "n": 30, | |
| "base_accuracy": 0.8666666666666667, | |
| "sentinel_overall_accuracy": 0.9, | |
| "sentinel_unflagged_accuracy": 0.9285714285714286, | |
| "sentinel_unflagged_n": 28, | |
| "sentinel_flag_rate": 0.06666666666666667, | |
| "sentinel_flag_rate_on_its_wrong": 0.3333333333333333, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "B_multi_turn/all": { | |
| "n": 100, | |
| "base_accuracy": 0.43, | |
| "sentinel_overall_accuracy": 0.38, | |
| "sentinel_unflagged_accuracy": 0.8484848484848485, | |
| "sentinel_unflagged_n": 33, | |
| "sentinel_flag_rate": 0.67, | |
| "sentinel_flag_rate_on_its_wrong": 0.9193548387096774, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "C_mixed_chat/fact": { | |
| "n": 70, | |
| "base_accuracy": 0.2571428571428571, | |
| "sentinel_overall_accuracy": 0.18571428571428572, | |
| "sentinel_unflagged_accuracy": 0.0, | |
| "sentinel_unflagged_n": 1, | |
| "sentinel_flag_rate": 0.9857142857142858, | |
| "sentinel_flag_rate_on_its_wrong": 0.9824561403508771, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "C_mixed_chat/math": { | |
| "n": 30, | |
| "base_accuracy": 0.7333333333333333, | |
| "sentinel_overall_accuracy": 0.8666666666666667, | |
| "sentinel_unflagged_accuracy": 0.75, | |
| "sentinel_unflagged_n": 12, | |
| "sentinel_flag_rate": 0.6, | |
| "sentinel_flag_rate_on_its_wrong": 0.25, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "C_mixed_chat/all": { | |
| "n": 100, | |
| "base_accuracy": 0.4, | |
| "sentinel_overall_accuracy": 0.39, | |
| "sentinel_unflagged_accuracy": 0.6923076923076923, | |
| "sentinel_unflagged_n": 13, | |
| "sentinel_flag_rate": 0.87, | |
| "sentinel_flag_rate_on_its_wrong": 0.9344262295081968, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "all/fact": { | |
| "n": 210, | |
| "base_accuracy": 0.2619047619047619, | |
| "sentinel_overall_accuracy": 0.14761904761904762, | |
| "sentinel_unflagged_accuracy": 0.35714285714285715, | |
| "sentinel_unflagged_n": 14, | |
| "sentinel_flag_rate": 0.9333333333333333, | |
| "sentinel_flag_rate_on_its_wrong": 0.9497206703910615, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| }, | |
| "all/math": { | |
| "n": 90, | |
| "base_accuracy": 0.8222222222222222, | |
| "sentinel_overall_accuracy": 0.9222222222222223, | |
| "sentinel_unflagged_accuracy": 0.9230769230769231, | |
| "sentinel_unflagged_n": 65, | |
| "sentinel_flag_rate": 0.2777777777777778, | |
| "sentinel_flag_rate_on_its_wrong": 0.2857142857142857, | |
| "sentinel_unsure_anywhere_not_leading": 0 | |
| } | |
| }, | |
| "earlier_turns_flagged_sentinel": 2, | |
| "all_pass": true, | |
| "search_when_unsure_reported": { | |
| "index_download": { | |
| "ok": true, | |
| "err": null, | |
| "seconds": 358, | |
| "java_home": "/tmp/scratch_wiki/jdk/jdk-21.0.13+9", | |
| "index_dir": "/tmp/scratch_wiki/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6", | |
| "jar": "/tmp/scratch_wiki/anserini-1.1.1-fatjar.jar" | |
| }, | |
| "flagged_fact_conversations": 196, | |
| "probe_hits": "4/5", | |
| "guess_accuracy_on_flagged": 0.1326530612244898, | |
| "search_accuracy_on_flagged": 0.1683673469387755 | |
| }, | |
| "seconds": 764, | |
| "env": { | |
| "model": "allenai/Olmo-3-7B-Instruct", | |
| "revision": "6e5971d9eba42665f5bd5a0fcf047f299ce1dccc", | |
| "vllm": "0.31.0", | |
| "torch": "2.13.0+cu130", | |
| "gpu": "NVIDIA A10G", | |
| "conversations_sha256": "d3e5aa89b14599fab64e5aab89e419add90cfc591a1bee31c5e10539a6e33a07", | |
| "n_conversations": 350, | |
| "adapter_sha256": { | |
| "README.md": "88eb49c0bd7a9d88fd9bb7275a49c6d29c6d6ff4716cf4055604240041ce093b", | |
| "adapter_config.json": "536fb661f743f58fd8309c91045e7c1784bab9bbdd469d7731e3c914159ad86b", | |
| "adapter_model.safetensors": "d1a21fd949cce0f8beb8f4575963ada1558aab35b22fff54a224c9472cc0768b" | |
| }, | |
| "early_max_tokens": 512, | |
| "last_max_tokens": 1024 | |
| } | |
| } |