Text Classification
Transformers
Safetensors
English
modernbert
refusal-classification
refusal-detection
ai-safety
ai-alignment
heretic
text-embeddings-inference
Instructions to use saidutta69/vantablack with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use saidutta69/vantablack with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="saidutta69/vantablack")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("saidutta69/vantablack") model = AutoModelForSequenceClassification.from_pretrained("saidutta69/vantablack", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download metrics.json from saidutta69/vantablack: direct link, hf CLI and curl.
- Browser
- Download file 8.52 kB
-
https://huggingface.co/saidutta69/vantablack/resolve/main/metrics.json
- Command line
-
hf download hf://saidutta69/vantablack/metrics.json
-
curl -L -o metrics.json https://huggingface.co/saidutta69/vantablack/resolve/main/metrics.json
8.52 kB
| { | |
| "checkpoint": "runs/mbert-large-v4-calibrated", | |
| "calibration": { | |
| "temperature": 1.0000000000000004, | |
| "bias": [ | |
| 1.0, | |
| 0.0, | |
| 0.0, | |
| 0.0, | |
| 0.9 | |
| ] | |
| }, | |
| "val_raw": { | |
| "accuracy": 0.9775280898876404, | |
| "macro_f1": 0.9775743091117549, | |
| "macro_f1_supported": 0.9775743091117549, | |
| "weighted_f1": 0.9775158976678895, | |
| "nll": 0.1288159340620041, | |
| "ece": 0.011444486660903795, | |
| "brier": 0.04359452426433563, | |
| "aurc": 0.012627429970769228, | |
| "mean_conf": 0.9716665148735046, | |
| "confident_error_rate": 0.8941255807876587, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.49887640449438203, | |
| "accuracy": 0.990990990990991, | |
| "confidence_cutoff": 0.9896456003189087 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9831460674157303, | |
| "confidence_cutoff": 0.9393740892410278 | |
| }, | |
| "0.9": { | |
| "coverage": 0.898876404494382, | |
| "accuracy": 0.985, | |
| "confidence_cutoff": 0.9332247376441956 | |
| } | |
| }, | |
| "per_class": { | |
| "compliant": { | |
| "precision": 1.0, | |
| "recall": 0.946236559139785, | |
| "f1": 0.9723756906077349, | |
| "support": 93 | |
| }, | |
| "full_refusal": { | |
| "precision": 0.9883720930232558, | |
| "recall": 0.9659090909090909, | |
| "f1": 0.9770114942528736, | |
| "support": 88 | |
| }, | |
| "partial_refusal": { | |
| "precision": 0.945054945054945, | |
| "recall": 0.9772727272727273, | |
| "f1": 0.9608938547486033, | |
| "support": 88 | |
| }, | |
| "capability_limitation": { | |
| "precision": 0.9887640449438202, | |
| "recall": 1.0, | |
| "f1": 0.9943502824858756, | |
| "support": 88 | |
| }, | |
| "clarification": { | |
| "precision": 0.967032967032967, | |
| "recall": 1.0, | |
| "f1": 0.9832402234636871, | |
| "support": 88 | |
| } | |
| }, | |
| "confusion": [ | |
| [ | |
| 88, | |
| 0, | |
| 2, | |
| 1, | |
| 2 | |
| ], | |
| [ | |
| 0, | |
| 85, | |
| 3, | |
| 0, | |
| 0 | |
| ], | |
| [ | |
| 0, | |
| 1, | |
| 86, | |
| 0, | |
| 1 | |
| ], | |
| [ | |
| 0, | |
| 0, | |
| 0, | |
| 88, | |
| 0 | |
| ], | |
| [ | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| 88 | |
| ] | |
| ], | |
| "support": 445 | |
| }, | |
| "val_calibrated": { | |
| "accuracy": 0.9797752808988764, | |
| "macro_f1": 0.9797832226111749, | |
| "macro_f1_supported": 0.9797832226111749, | |
| "weighted_f1": 0.9797634333461278, | |
| "nll": 0.12827310616382437, | |
| "ece": 0.03314757731706912, | |
| "brier": 0.043282836501022375, | |
| "aurc": 0.010907583758805439, | |
| "mean_conf": 0.9682135336223163, | |
| "confident_error_rate": 0.9188248039158842, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.49887640449438203, | |
| "accuracy": 0.9819819819819819, | |
| "confidence_cutoff": 0.9916877198340319 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9831460674157303, | |
| "confidence_cutoff": 0.913997891114871 | |
| }, | |
| "0.9": { | |
| "coverage": 0.898876404494382, | |
| "accuracy": 0.985, | |
| "confidence_cutoff": 0.9059301601030256 | |
| } | |
| }, | |
| "per_class": { | |
| "compliant": { | |
| "precision": 1.0, | |
| "recall": 0.956989247311828, | |
| "f1": 0.978021978021978, | |
| "support": 93 | |
| }, | |
| "full_refusal": { | |
| "precision": 0.9883720930232558, | |
| "recall": 0.9659090909090909, | |
| "f1": 0.9770114942528736, | |
| "support": 88 | |
| }, | |
| "partial_refusal": { | |
| "precision": 0.9555555555555556, | |
| "recall": 0.9772727272727273, | |
| "f1": 0.9662921348314608, | |
| "support": 88 | |
| }, | |
| "capability_limitation": { | |
| "precision": 0.9887640449438202, | |
| "recall": 1.0, | |
| "f1": 0.9943502824858756, | |
| "support": 88 | |
| }, | |
| "clarification": { | |
| "precision": 0.967032967032967, | |
| "recall": 1.0, | |
| "f1": 0.9832402234636871, | |
| "support": 88 | |
| } | |
| }, | |
| "confusion": [ | |
| [ | |
| 89, | |
| 0, | |
| 1, | |
| 1, | |
| 2 | |
| ], | |
| [ | |
| 0, | |
| 85, | |
| 3, | |
| 0, | |
| 0 | |
| ], | |
| [ | |
| 0, | |
| 1, | |
| 86, | |
| 0, | |
| 1 | |
| ], | |
| [ | |
| 0, | |
| 0, | |
| 0, | |
| 88, | |
| 0 | |
| ], | |
| [ | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| 88 | |
| ] | |
| ], | |
| "support": 445 | |
| }, | |
| "test_raw": { | |
| "accuracy": 0.9910112359550561, | |
| "macro_f1": 0.9910740397730361, | |
| "macro_f1_supported": 0.9910740397730361, | |
| "weighted_f1": 0.9909994244508226, | |
| "nll": 0.06652859598398209, | |
| "ece": 0.01684493764062944, | |
| "brier": 0.017338627949357033, | |
| "aurc": 0.008423663643542165, | |
| "mean_conf": 0.9749369025230408, | |
| "confident_error_rate": 0.8797094821929932, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.49887640449438203, | |
| "accuracy": 0.990990990990991, | |
| "confidence_cutoff": 0.989537239074707 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9915730337078652, | |
| "confidence_cutoff": 0.9396082162857056 | |
| }, | |
| "0.9": { | |
| "coverage": 0.898876404494382, | |
| "accuracy": 0.9925, | |
| "confidence_cutoff": 0.9324463605880737 | |
| } | |
| }, | |
| "per_class": { | |
| "compliant": { | |
| "precision": 0.989247311827957, | |
| "recall": 0.9787234042553191, | |
| "f1": 0.983957219251337, | |
| "support": 94 | |
| }, | |
| "full_refusal": { | |
| "precision": 1.0, | |
| "recall": 0.9886363636363636, | |
| "f1": 0.9942857142857142, | |
| "support": 88 | |
| }, | |
| "partial_refusal": { | |
| "precision": 0.9883720930232558, | |
| "recall": 0.9883720930232558, | |
| "f1": 0.9883720930232558, | |
| "support": 86 | |
| }, | |
| "capability_limitation": { | |
| "precision": 0.9885057471264368, | |
| "recall": 1.0, | |
| "f1": 0.9942196531791908, | |
| "support": 86 | |
| }, | |
| "clarification": { | |
| "precision": 0.9891304347826086, | |
| "recall": 1.0, | |
| "f1": 0.994535519125683, | |
| "support": 91 | |
| } | |
| }, | |
| "confusion": [ | |
| [ | |
| 92, | |
| 0, | |
| 0, | |
| 1, | |
| 1 | |
| ], | |
| [ | |
| 0, | |
| 87, | |
| 1, | |
| 0, | |
| 0 | |
| ], | |
| [ | |
| 1, | |
| 0, | |
| 85, | |
| 0, | |
| 0 | |
| ], | |
| [ | |
| 0, | |
| 0, | |
| 0, | |
| 86, | |
| 0 | |
| ], | |
| [ | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| 91 | |
| ] | |
| ], | |
| "support": 445 | |
| }, | |
| "test_calibrated": { | |
| "accuracy": 0.9910112359550561, | |
| "macro_f1": 0.9910740397730361, | |
| "macro_f1_supported": 0.9910740397730361, | |
| "weighted_f1": 0.9909994244508226, | |
| "nll": 0.06978580453913351, | |
| "ece": 0.022631346186655556, | |
| "brier": 0.018303621149644102, | |
| "aurc": 0.004501822525535942, | |
| "mean_conf": 0.971843404608016, | |
| "confident_error_rate": 0.8869774945028713, | |
| "selective": { | |
| "0.5": { | |
| "coverage": 0.49887640449438203, | |
| "accuracy": 0.9954954954954955, | |
| "confidence_cutoff": 0.9919190574804833 | |
| }, | |
| "0.8": { | |
| "coverage": 0.8, | |
| "accuracy": 0.9915730337078652, | |
| "confidence_cutoff": 0.9185372280015965 | |
| }, | |
| "0.9": { | |
| "coverage": 0.898876404494382, | |
| "accuracy": 0.9925, | |
| "confidence_cutoff": 0.9058025500487228 | |
| } | |
| }, | |
| "per_class": { | |
| "compliant": { | |
| "precision": 0.989247311827957, | |
| "recall": 0.9787234042553191, | |
| "f1": 0.983957219251337, | |
| "support": 94 | |
| }, | |
| "full_refusal": { | |
| "precision": 1.0, | |
| "recall": 0.9886363636363636, | |
| "f1": 0.9942857142857142, | |
| "support": 88 | |
| }, | |
| "partial_refusal": { | |
| "precision": 0.9883720930232558, | |
| "recall": 0.9883720930232558, | |
| "f1": 0.9883720930232558, | |
| "support": 86 | |
| }, | |
| "capability_limitation": { | |
| "precision": 0.9885057471264368, | |
| "recall": 1.0, | |
| "f1": 0.9942196531791908, | |
| "support": 86 | |
| }, | |
| "clarification": { | |
| "precision": 0.9891304347826086, | |
| "recall": 1.0, | |
| "f1": 0.994535519125683, | |
| "support": 91 | |
| } | |
| }, | |
| "confusion": [ | |
| [ | |
| 92, | |
| 0, | |
| 0, | |
| 1, | |
| 1 | |
| ], | |
| [ | |
| 0, | |
| 87, | |
| 1, | |
| 0, | |
| 0 | |
| ], | |
| [ | |
| 1, | |
| 0, | |
| 85, | |
| 0, | |
| 0 | |
| ], | |
| [ | |
| 0, | |
| 0, | |
| 0, | |
| 86, | |
| 0 | |
| ], | |
| [ | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| 91 | |
| ] | |
| ], | |
| "support": 445 | |
| } | |
| } |