| { | |
| "experiments": [ | |
| { | |
| "branch": "orx/threshold-conversion-baseline-theory-and-simplex", | |
| "commit": "9aa1d7906cd301302adda9e77fd167694f7263e5", | |
| "device": "auto (text=cpu, vision=mps if available, digits=cpu)", | |
| "id": "8a044950-bedc-4084-8507-75e4555d9650", | |
| "run": "cff2a16d-f190-42ed-8510-38d21ec359ad", | |
| "slug": "threshold-conversion-baseline-theory-and-simplex", | |
| "tracks": [ | |
| "theory", | |
| "digits" | |
| ] | |
| }, | |
| { | |
| "branch": "orx/real-text-classifiers-2-3-and-6-classes", | |
| "commit": "48269177ba20f5b97e496ab5bcd71c5513461f6b", | |
| "device": "auto (text=cpu, vision=mps if available, digits=cpu)", | |
| "id": "9512af43-d119-4b35-b9d0-b11b727b28e3", | |
| "run": "18fb96ec-2a40-43dc-ac78-6343bdc067aa", | |
| "slug": "real-text-classifiers-2-3-and-6-classes", | |
| "tracks": [ | |
| "text" | |
| ] | |
| }, | |
| { | |
| "branch": "orx/1000-class-vision-vit-b-16-on-cifar-10", | |
| "commit": "640f83fde36f5d31a0bcf1ab304bf96c971a22a0", | |
| "device": "auto (text=cpu, vision=mps if available, digits=cpu)", | |
| "id": "493a2ee5-43dd-4e01-b541-3aedf0bbab4f", | |
| "run": "a5f6fc9f-88fb-415e-92b1-fc064fa95d96", | |
| "slug": "1000-class-vision-vit-b-16-on-cifar-10", | |
| "tracks": [ | |
| "vision" | |
| ] | |
| }, | |
| { | |
| "branch": "orx/can-a-fitted-fixed-conversion-replace-the-adapti", | |
| "commit": "76f0fbed859a44c648e22af7e0d88095d9fc00ef", | |
| "device": "auto (text=cpu, vision=mps if available, digits=cpu)", | |
| "id": "3ab3fa4c-eab3-4cc1-96f9-126f68d0a24a", | |
| "run": "6af95ffd-982c-45ca-b87c-65f817fb1aef", | |
| "slug": "can-a-fitted-fixed-conversion-replace-the-adapti", | |
| "tracks": [ | |
| "text", | |
| "conversion_fit" | |
| ] | |
| }, | |
| { | |
| "branch": "orx/tabular-classifiers-and-the-max-f1-0-5-conversio", | |
| "commit": "72cceab1c53dc3669430cc937d2b316a08dbb51a", | |
| "device": "auto (text=cpu, vision=mps if available, digits=cpu)", | |
| "id": "b117d2f9-70a2-4eb7-82cd-98cd1f99e632", | |
| "run": "576b9f04-e295-4300-b42b-de57f96a54bb", | |
| "slug": "tabular-classifiers-and-the-max-f1-0-5-conversio", | |
| "tracks": [ | |
| "tabular" | |
| ] | |
| } | |
| ], | |
| "github": { | |
| "fork": "https://github.com/jomasego/python-multiagent-system", | |
| "upstream": "https://github.com/isllabres/python-multiagent-system" | |
| }, | |
| "huggingface": { | |
| "bucket": "https://huggingface.co/buckets/jomasego/repro-threshold-conversion-artifacts", | |
| "datasets": [ | |
| "https://huggingface.co/datasets/cardiffnlp/tweet_eval", | |
| "https://huggingface.co/datasets/dair-ai/emotion", | |
| "https://huggingface.co/datasets/uoft-cs/cifar10", | |
| "https://huggingface.co/datasets/stanfordnlp/sst2" | |
| ], | |
| "job": { | |
| "attempted": true, | |
| "error": "Hugging Face job submit failed (402): Pre-paid credit balance is insufficient - add more credits to your account to use Jobs.", | |
| "status": "blocked" | |
| }, | |
| "models": [ | |
| "https://huggingface.co/cardiffnlp/twitter-roberta-base-sentiment-latest", | |
| "https://huggingface.co/bhadresh-savani/distilbert-base-uncased-emotion", | |
| "https://huggingface.co/google/vit-base-patch16-224", | |
| "https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english" | |
| ] | |
| }, | |
| "metrics": { | |
| "binary_sst2": { | |
| "C": 2, | |
| "accuracy_argmax": 0.9105504587155964, | |
| "coverage_0.5": 1.0, | |
| "delta_0.5": 0.0, | |
| "ece_15": 0.0800265887100546, | |
| "exact_adaptive": true, | |
| "mean_max_prob": 0.9833779635937905, | |
| "median_max_prob": 0.9994933986842978, | |
| "n": 872, | |
| "name": "binary control: DistilBERT SST-2", | |
| "selective_accuracy": 0.9105504587155964 | |
| }, | |
| "conversion_fit": { | |
| "binary_sst2": { | |
| "chance": 0.5, | |
| "name": "binary_sst2", | |
| "test_accuracy": 1.0, | |
| "train_accuracy": 1.0 | |
| }, | |
| "emotion6": { | |
| "chance": 0.16666666666666666, | |
| "name": "emotion6", | |
| "test_accuracy": 0.9983333333333333, | |
| "train_accuracy": 0.995 | |
| }, | |
| "sentiment3": { | |
| "chance": 0.3333333333333333, | |
| "name": "sentiment3", | |
| "test_accuracy": 0.9933333333333333, | |
| "train_accuracy": 0.9914285714285714 | |
| } | |
| }, | |
| "digits10": { | |
| "C": 10, | |
| "accuracy_argmax": 0.9722222222222222, | |
| "coverage_0.5": 0.9925925925925926, | |
| "delta_0.5": 0.007407407407407408, | |
| "ece_15": 0.022187818228840778, | |
| "exact_adaptive": true, | |
| "mean_max_prob": 0.9523571975284499, | |
| "median_max_prob": 0.9956063114382108, | |
| "n": 540, | |
| "name": "10-class tabular: logistic regression on sklearn digits", | |
| "selective_accuracy": 0.9776119402985075 | |
| }, | |
| "emotion6": { | |
| "C": 6, | |
| "accuracy_argmax": 0.927, | |
| "coverage_0.5": 0.998, | |
| "delta_0.5": 0.002, | |
| "ece_15": 0.038544882350143965, | |
| "exact_adaptive": true, | |
| "mean_max_prob": 0.9647051939280312, | |
| "median_max_prob": 0.9985358442899119, | |
| "n": 2000, | |
| "name": "6-class emotion: DistilBERT emotion", | |
| "selective_accuracy": 0.9268537074148296 | |
| }, | |
| "sentiment3": { | |
| "C": 3, | |
| "accuracy_argmax": 0.715, | |
| "coverage_0.5": 0.9535, | |
| "delta_0.5": 0.0465, | |
| "ece_15": 0.05731805985703993, | |
| "exact_adaptive": true, | |
| "mean_max_prob": 0.755950959971354, | |
| "median_max_prob": 0.7711395904964865, | |
| "n": 2000, | |
| "name": "3-class sentiment: Twitter-RoBERTa", | |
| "selective_accuracy": 0.723649711588883 | |
| }, | |
| "tabular": { | |
| "all_exact": true, | |
| "binary_tau_range": [ | |
| 0.4818890118886161, | |
| 0.9984020798385944 | |
| ], | |
| "max_delta": 0.998, | |
| "max_tau_spread": 0.9999999980000001, | |
| "min_argmax_agreement": 0.769, | |
| "n_pairs": 30 | |
| }, | |
| "theory": { | |
| "closed_form": { | |
| "10": 0.98046875, | |
| "2": 0.0, | |
| "20": 0.9999618530273438, | |
| "3": 0.25, | |
| "4": 0.5, | |
| "5": 0.6875, | |
| "6": 0.8125 | |
| }, | |
| "uniform_delta": { | |
| "10": 0.980625, | |
| "2": 0.0, | |
| "20": 0.99995, | |
| "3": 0.2486, | |
| "4": 0.49455, | |
| "5": 0.68615, | |
| "6": 0.809025 | |
| }, | |
| "whiteboard": { | |
| "argmax": 1, | |
| "dot": 0.41400000000000003, | |
| "dot_threshold_pred": 0, | |
| "hadamard_argmax": 1, | |
| "p": [ | |
| 0.07, | |
| 0.93 | |
| ], | |
| "renormalized": [ | |
| 0.10144927536231883, | |
| 0.8985507246376813 | |
| ], | |
| "renormalized_argmax": 1, | |
| "w": [ | |
| 0.6, | |
| 0.4 | |
| ] | |
| } | |
| }, | |
| "vit1000": { | |
| "C": 1000, | |
| "coverage_0.5": 0.49, | |
| "delta_0.5": 0.51, | |
| "exact_adaptive": true, | |
| "mean_max_prob": 0.5228046251597489, | |
| "median_max_prob": 0.4888720744666699, | |
| "n": 1000, | |
| "name": "1000-class vision: ViT-B/16 (ImageNet head on CIFAR-10)" | |
| } | |
| }, | |
| "project": { | |
| "id": "b588d9dc-8af1-49ae-b1db-fdbeb45e3dd3", | |
| "name": "python-multiagent-system" | |
| }, | |
| "question": "Can any multiclass threshold curve be converted to a 0.5-threshold rule equivalent to argmax, preserving maximum F1?", | |
| "verdict": "True as a per-threshold or per-class affine reparameterization; false as one fixed 0.5 rule for multiclass argmax. Max-F1 conversion preserves the selected threshold exactly but cannot improve F1 and does not replace argmax." | |
| } |
Xet Storage Details
- Size:
- 7.32 kB
- Xet hash:
- 7b6441f2a3632116184ddd52310aa5633b2867bf144546411b7d093ab4941998
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.