jomasego's picture
download
raw
7.32 kB
{
"experiments": [
{
"branch": "orx/threshold-conversion-baseline-theory-and-simplex",
"commit": "9aa1d7906cd301302adda9e77fd167694f7263e5",
"device": "auto (text=cpu, vision=mps if available, digits=cpu)",
"id": "8a044950-bedc-4084-8507-75e4555d9650",
"run": "cff2a16d-f190-42ed-8510-38d21ec359ad",
"slug": "threshold-conversion-baseline-theory-and-simplex",
"tracks": [
"theory",
"digits"
]
},
{
"branch": "orx/real-text-classifiers-2-3-and-6-classes",
"commit": "48269177ba20f5b97e496ab5bcd71c5513461f6b",
"device": "auto (text=cpu, vision=mps if available, digits=cpu)",
"id": "9512af43-d119-4b35-b9d0-b11b727b28e3",
"run": "18fb96ec-2a40-43dc-ac78-6343bdc067aa",
"slug": "real-text-classifiers-2-3-and-6-classes",
"tracks": [
"text"
]
},
{
"branch": "orx/1000-class-vision-vit-b-16-on-cifar-10",
"commit": "640f83fde36f5d31a0bcf1ab304bf96c971a22a0",
"device": "auto (text=cpu, vision=mps if available, digits=cpu)",
"id": "493a2ee5-43dd-4e01-b541-3aedf0bbab4f",
"run": "a5f6fc9f-88fb-415e-92b1-fc064fa95d96",
"slug": "1000-class-vision-vit-b-16-on-cifar-10",
"tracks": [
"vision"
]
},
{
"branch": "orx/can-a-fitted-fixed-conversion-replace-the-adapti",
"commit": "76f0fbed859a44c648e22af7e0d88095d9fc00ef",
"device": "auto (text=cpu, vision=mps if available, digits=cpu)",
"id": "3ab3fa4c-eab3-4cc1-96f9-126f68d0a24a",
"run": "6af95ffd-982c-45ca-b87c-65f817fb1aef",
"slug": "can-a-fitted-fixed-conversion-replace-the-adapti",
"tracks": [
"text",
"conversion_fit"
]
},
{
"branch": "orx/tabular-classifiers-and-the-max-f1-0-5-conversio",
"commit": "72cceab1c53dc3669430cc937d2b316a08dbb51a",
"device": "auto (text=cpu, vision=mps if available, digits=cpu)",
"id": "b117d2f9-70a2-4eb7-82cd-98cd1f99e632",
"run": "576b9f04-e295-4300-b42b-de57f96a54bb",
"slug": "tabular-classifiers-and-the-max-f1-0-5-conversio",
"tracks": [
"tabular"
]
}
],
"github": {
"fork": "https://github.com/jomasego/python-multiagent-system",
"upstream": "https://github.com/isllabres/python-multiagent-system"
},
"huggingface": {
"bucket": "https://huggingface.co/buckets/jomasego/repro-threshold-conversion-artifacts",
"datasets": [
"https://huggingface.co/datasets/cardiffnlp/tweet_eval",
"https://huggingface.co/datasets/dair-ai/emotion",
"https://huggingface.co/datasets/uoft-cs/cifar10",
"https://huggingface.co/datasets/stanfordnlp/sst2"
],
"job": {
"attempted": true,
"error": "Hugging Face job submit failed (402): Pre-paid credit balance is insufficient - add more credits to your account to use Jobs.",
"status": "blocked"
},
"models": [
"https://huggingface.co/cardiffnlp/twitter-roberta-base-sentiment-latest",
"https://huggingface.co/bhadresh-savani/distilbert-base-uncased-emotion",
"https://huggingface.co/google/vit-base-patch16-224",
"https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english"
]
},
"metrics": {
"binary_sst2": {
"C": 2,
"accuracy_argmax": 0.9105504587155964,
"coverage_0.5": 1.0,
"delta_0.5": 0.0,
"ece_15": 0.0800265887100546,
"exact_adaptive": true,
"mean_max_prob": 0.9833779635937905,
"median_max_prob": 0.9994933986842978,
"n": 872,
"name": "binary control: DistilBERT SST-2",
"selective_accuracy": 0.9105504587155964
},
"conversion_fit": {
"binary_sst2": {
"chance": 0.5,
"name": "binary_sst2",
"test_accuracy": 1.0,
"train_accuracy": 1.0
},
"emotion6": {
"chance": 0.16666666666666666,
"name": "emotion6",
"test_accuracy": 0.9983333333333333,
"train_accuracy": 0.995
},
"sentiment3": {
"chance": 0.3333333333333333,
"name": "sentiment3",
"test_accuracy": 0.9933333333333333,
"train_accuracy": 0.9914285714285714
}
},
"digits10": {
"C": 10,
"accuracy_argmax": 0.9722222222222222,
"coverage_0.5": 0.9925925925925926,
"delta_0.5": 0.007407407407407408,
"ece_15": 0.022187818228840778,
"exact_adaptive": true,
"mean_max_prob": 0.9523571975284499,
"median_max_prob": 0.9956063114382108,
"n": 540,
"name": "10-class tabular: logistic regression on sklearn digits",
"selective_accuracy": 0.9776119402985075
},
"emotion6": {
"C": 6,
"accuracy_argmax": 0.927,
"coverage_0.5": 0.998,
"delta_0.5": 0.002,
"ece_15": 0.038544882350143965,
"exact_adaptive": true,
"mean_max_prob": 0.9647051939280312,
"median_max_prob": 0.9985358442899119,
"n": 2000,
"name": "6-class emotion: DistilBERT emotion",
"selective_accuracy": 0.9268537074148296
},
"sentiment3": {
"C": 3,
"accuracy_argmax": 0.715,
"coverage_0.5": 0.9535,
"delta_0.5": 0.0465,
"ece_15": 0.05731805985703993,
"exact_adaptive": true,
"mean_max_prob": 0.755950959971354,
"median_max_prob": 0.7711395904964865,
"n": 2000,
"name": "3-class sentiment: Twitter-RoBERTa",
"selective_accuracy": 0.723649711588883
},
"tabular": {
"all_exact": true,
"binary_tau_range": [
0.4818890118886161,
0.9984020798385944
],
"max_delta": 0.998,
"max_tau_spread": 0.9999999980000001,
"min_argmax_agreement": 0.769,
"n_pairs": 30
},
"theory": {
"closed_form": {
"10": 0.98046875,
"2": 0.0,
"20": 0.9999618530273438,
"3": 0.25,
"4": 0.5,
"5": 0.6875,
"6": 0.8125
},
"uniform_delta": {
"10": 0.980625,
"2": 0.0,
"20": 0.99995,
"3": 0.2486,
"4": 0.49455,
"5": 0.68615,
"6": 0.809025
},
"whiteboard": {
"argmax": 1,
"dot": 0.41400000000000003,
"dot_threshold_pred": 0,
"hadamard_argmax": 1,
"p": [
0.07,
0.93
],
"renormalized": [
0.10144927536231883,
0.8985507246376813
],
"renormalized_argmax": 1,
"w": [
0.6,
0.4
]
}
},
"vit1000": {
"C": 1000,
"coverage_0.5": 0.49,
"delta_0.5": 0.51,
"exact_adaptive": true,
"mean_max_prob": 0.5228046251597489,
"median_max_prob": 0.4888720744666699,
"n": 1000,
"name": "1000-class vision: ViT-B/16 (ImageNet head on CIFAR-10)"
}
},
"project": {
"id": "b588d9dc-8af1-49ae-b1db-fdbeb45e3dd3",
"name": "python-multiagent-system"
},
"question": "Can any multiclass threshold curve be converted to a 0.5-threshold rule equivalent to argmax, preserving maximum F1?",
"verdict": "True as a per-threshold or per-class affine reparameterization; false as one fixed 0.5 rule for multiclass argmax. Max-F1 conversion preserves the selected threshold exactly but cannot improve F1 and does not replace argmax."
}

Xet Storage Details

Size:
7.32 kB
·
Xet hash:
7b6441f2a3632116184ddd52310aa5633b2867bf144546411b7d093ab4941998

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.