{ "project": "Next Ask", "profile": "full", "seed": 73, "device": { "python": "3.13.16", "platform": "Linux-6.6.159+-x86_64-with-glibc2.39", "cpu_count": 12, "numpy": "2.4.6", "torch": "2.14.1+cu130", "cuda": true, "gpu": "NVIDIA L4", "colab": true }, "data": { "name": "Optical Recognition of Handwritten Digits (8x8)", "source": "UCI ML Repository id 80 via sklearn.datasets.load_digits", "license": "CC BY 4.0", "sklearn_version": "1.6.1", "n_total": 1797, "split": { "pool": 1237, "validation": 200, "test": 360, "method": "stratified train_test_split, random_state=73; test only for final reporting" } }, "classifier": { "type": "multinomial logistic regression, full-batch GD from zeros", "hyper": { "steps": 150, "lr": 1.0, "l2": 0.001 }, "hyper_selection": { "split": "validation", "grid": [ { "hyper": { "steps": 100, "lr": 0.5, "l2": 0.001 }, "val_accuracy": 0.7816666666666666 }, { "hyper": { "steps": 100, "lr": 0.5, "l2": 0.01 }, "val_accuracy": 0.775 }, { "hyper": { "steps": 100, "lr": 1.0, "l2": 0.001 }, "val_accuracy": 0.7966666666666667 }, { "hyper": { "steps": 100, "lr": 1.0, "l2": 0.01 }, "val_accuracy": 0.79 }, { "hyper": { "steps": 150, "lr": 0.5, "l2": 0.001 }, "val_accuracy": 0.7949999999999999 }, { "hyper": { "steps": 150, "lr": 0.5, "l2": 0.01 }, "val_accuracy": 0.7833333333333333 }, { "hyper": { "steps": 150, "lr": 1.0, "l2": 0.001 }, "val_accuracy": 0.8083333333333335 }, { "hyper": { "steps": 150, "lr": 1.0, "l2": 0.01 }, "val_accuracy": 0.7949999999999999 } ] } }, "protocol": { "seeds": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ], "initial_labels": 10, "batch": 10, "budget": 200, "pairing": "all strategies share the same initial labeled set per seed" }, "label_axis": [ 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190, 200 ], "results": { "random": { "mean_curve": [ 0.4972222222222221, 0.6508333333333334, 0.7319444444444445, 0.8058333333333334, 0.8386111111111111, 0.8577777777777778, 0.875, 0.8733333333333334, 0.8822222222222222, 0.8863888888888889, 0.891388888888889, 0.8952777777777777, 0.9038888888888887, 0.9038888888888892, 0.9130555555555556, 0.9152777777777779, 0.9175000000000001, 0.9233333333333335, 0.9252777777777779, 0.9283333333333335 ], "std_curve": [ 0.059004080210452226, 0.06322175844391503, 0.04262327799642462, 0.04171201236225172, 0.03069287326295477, 0.020927564668708192, 0.024438130497691987, 0.025645975305741767, 0.02351122663277648, 0.019580870609530135, 0.01654819000956276, 0.016622627000419093, 0.016536528962374692, 0.015020562038439754, 0.010974859036188004, 0.012187450538044603, 0.015518308581626014, 0.013217085837078126, 0.010573814617041271, 0.009922539499717643 ], "aulc_mean": 0.8508194444444446, "aulc_std": 0.011463921196411408, "final_accuracy_mean": 0.9283333333333335, "labels_to_90pct_per_seed": [ 120, 90, 130, 110, 80, 100, 170, 70, 130, 150 ] }, "uncertainty": { "mean_curve": [ 0.4972222222222221, 0.6608333333333334, 0.7691666666666667, 0.836111111111111, 0.865, 0.8869444444444443, 0.9080555555555556, 0.9172222222222223, 0.9241666666666666, 0.9294444444444444, 0.9358333333333333, 0.9411111111111111, 0.9413888888888889, 0.943611111111111, 0.9447222222222222, 0.9497222222222221, 0.9533333333333334, 0.9541666666666666, 0.9588888888888889, 0.9569444444444445 ], "std_curve": [ 0.059004080210452226, 0.04963524360177361, 0.06437556184580423, 0.04897089082453114, 0.032231799343022866, 0.02629515516564194, 0.012695994313362711, 0.010815512407739881, 0.01061750825200912, 0.011466537466972383, 0.009972957261675673, 0.007222222222222197, 0.008093779046851896, 0.007762882562490039, 0.010277777777777776, 0.006854979266251711, 0.006781419786518746, 0.006364688465216471, 0.004937885787397574, 0.006242279181734529 ], "aulc_mean": 0.8836944444444444, "aulc_std": 0.013546217184144036, "final_accuracy_mean": 0.9569444444444443, "labels_to_90pct_per_seed": [ 60, 70, 70, 70, 60, 80, 60, 80, 70, 60 ] }, "hybrid": { "mean_curve": [ 0.4972222222222221, 0.7036111111111111, 0.7972222222222223, 0.8483333333333333, 0.8736111111111111, 0.8933333333333333, 0.9052777777777778, 0.9175000000000001, 0.9244444444444445, 0.931388888888889, 0.9349999999999999, 0.9375, 0.9422222222222221, 0.9480555555555554, 0.9491666666666665, 0.9533333333333334, 0.9574999999999999, 0.9605555555555554, 0.9594444444444443, 0.9583333333333333 ], "std_curve": [ 0.059004080210452226, 0.04441057737427031, 0.027943947422131746, 0.03218388523991164, 0.011197580205970207, 0.006712803318663668, 0.012877029984999185, 0.013031893450822117, 0.007535922203472533, 0.008962369891612037, 0.009312808119022332, 0.011266277661485132, 0.009362388636862604, 0.007458734212276422, 0.006579288490181691, 0.006060395619242084, 0.006579288490181691, 0.0049378857873975624, 0.0033333333333333435, 0.005555555555555575 ], "aulc_mean": 0.8896527777777777, "aulc_std": 0.0066435507806675955, "final_accuracy_mean": 0.9583333333333333, "labels_to_90pct_per_seed": [ 60, 70, 60, 70, 70, 80, 80, 40, 80, 80 ] } }, "paired_aulc": { "hybrid_minus_random": { "aulc_diff_mean": 0.038833333333333366, "aulc_diff_std": 0.011321139954816477, "hybrid_wins": 10, "seeds": 10 }, "hybrid_minus_uncertainty": { "aulc_diff_mean": 0.005958333333333366, "aulc_diff_std": 0.010409525329840541, "hybrid_wins": 6, "seeds": 10 } }, "failure_examples": { "model": "hybrid seed 0 at full budget", "n_wrong": 17, "first": [ { "test_index": 648, "true": 2, "predicted": 3, "confidence": 0.4906110093411642 }, { "test_index": 1149, "true": 8, "predicted": 1, "confidence": 0.279524654801159 }, { "test_index": 1264, "true": 1, "predicted": 8, "confidence": 0.4017349208333158 }, { "test_index": 901, "true": 9, "predicted": 8, "confidence": 0.39549887000865813 }, { "test_index": 1595, "true": 7, "predicted": 4, "confidence": 0.3269425499797302 }, { "test_index": 498, "true": 7, "predicted": 9, "confidence": 0.3155363201045011 }, { "test_index": 492, "true": 6, "predicted": 8, "confidence": 0.4289399503435557 }, { "test_index": 746, "true": 4, "predicted": 7, "confidence": 0.513914544250967 }, { "test_index": 447, "true": 7, "predicted": 5, "confidence": 0.4440512466278117 }, { "test_index": 457, "true": 5, "predicted": 9, "confidence": 0.5211896895367181 }, { "test_index": 905, "true": 8, "predicted": 1, "confidence": 0.5462623831942506 }, { "test_index": 1095, "true": 4, "predicted": 9, "confidence": 0.38364309630663607 }, { "test_index": 857, "true": 7, "predicted": 3, "confidence": 0.5475465941519988 }, { "test_index": 719, "true": 9, "predicted": 1, "confidence": 0.5353181203934337 }, { "test_index": 1384, "true": 4, "predicted": 6, "confidence": 0.4296230000789044 }, { "test_index": 769, "true": 8, "predicted": 2, "confidence": 0.3883439380665226 }, { "test_index": 794, "true": 8, "predicted": 1, "confidence": 0.4823884399122324 } ] }, "tests": { "reload": { "returncode": 0, "seconds": 0.18, "process": "fresh python -E subprocess, cwd = temporary copy of the model folder", "passed": true, "max_abs_diff": 5.220997367771929e-08, "picks": [ 96, 36, 26, 7, 105 ], "expected_picks": [ 96, 36, 26, 7, 105 ], "n_samples": 16 }, "node_parity": { "cases": 4, "failed": 0, "maxDiff": 1.1102230246251565e-16, "results": [ { "name": "hybrid-10", "maxDiff": 1.1102230246251565e-16 }, { "name": "uncertainty-25", "maxDiff": 5.551115123125783e-17 }, { "name": "random-40", "maxDiff": 1.1102230246251565e-16 }, { "name": "hybrid-60", "maxDiff": 4.163336342344337e-17 } ], "ms": 114, "runtime": "node v22.23.4", "status": "passed" }, "browser": { "status": "skipped", "reason": "playwright not installed" } }, "not_run": [ "Human annotation study: the Space accepts user labels, but no user study was run." ] }