Next-Shift / evaluation.json
Lamapi's picture
Publish Next-Shift space
28b4298 verified
Raw History Blame Contribute Delete
9.68 kB
{
"project": "Next Shift",
"profile": "full",
"seed": 73,
"device": {
"python": "3.13.16",
"platform": "Linux-6.6.159+-x86_64-with-glibc2.39",
"cpu_count": 12,
"numpy": "2.4.6",
"torch": "2.14.1+cu130",
"cuda": true,
"gpu": "NVIDIA L4",
"colab": true
},
"data": {
"name": "Optical Recognition of Handwritten Digits",
"source": "UCI id 80 via sklearn.datasets.load_digits",
"license": "CC BY 4.0",
"sklearn_version": "1.6.1",
"target_environment": "synthetic: 16x16 upsampling, rotation ±25°, scale 0.8-1.05, grey background with weaker ink, illumination gradient, stripe texture, blur, noise",
"split": {
"source_train": 900,
"source_val": 150,
"target_pool": 350,
"target_dev": 100,
"target_test": 297,
"method": "stratified, random_state=73; source and target use disjoint digit samples"
}
},
"pretraining": {
"epochs": 40,
"best_epoch": 5,
"source_val_acc": 0.9733333333333334,
"seconds": 1.5,
"target_test_acc_source_only": 0.4006734006734007
},
"hyper": {
"ft_steps": 50,
"ft_lr": 0.05,
"lin_steps": 200,
"lin_lr": 0.1,
"rand_steps": 100,
"rand_lr": 0.001,
"noada_steps": 50,
"noada_lr": 0.05
},
"hyper_selection": {
"split": "target development (100 images)",
"dev_seeds": [
100,
101,
102
],
"next_shift": {
"50,0.05": 0.7033333333333335,
"50,0.1": 0.6933333333333334,
"50,0.3": 0.10000000000000002,
"100,0.05": 0.6933333333333334,
"100,0.1": 0.68,
"100,0.3": 0.10333333333333333,
"200,0.05": 0.68,
"200,0.1": 0.6733333333333333,
"200,0.3": 0.10666666666666667
},
"no_adabn": {
"50,0.05": 0.68,
"50,0.1": 0.6733333333333333,
"50,0.3": 0.12666666666666668,
"100,0.05": 0.6766666666666666,
"100,0.1": 0.6633333333333332,
"100,0.3": 0.10666666666666667,
"200,0.05": 0.6666666666666666,
"200,0.1": 0.6699999999999999,
"200,0.3": 0.11333333333333334
},
"frozen": {
"200,0.01": 0.6033333333333334,
"200,0.03": 0.6233333333333334,
"200,0.1": 0.63,
"500,0.01": 0.63,
"500,0.03": 0.6166666666666667,
"500,0.1": 0.6166666666666667
},
"random_init": {
"100,0.001": 0.5333333333333333,
"100,0.003": 0.5233333333333333,
"300,0.001": 0.52,
"300,0.003": 0.5166666666666667
}
},
"budget": "for each k all methods see the same labeled images; steps chosen per method on the development split",
"results": {
"next_shift": {
"k_per_class": [
0,
1,
5,
10
],
"mean": [
0.3939393939393939,
0.48888888888888893,
0.5878787878787878,
0.6612794612794612
],
"std": [
0.0,
0.03267207938237902,
0.020929656926491656,
0.009896928253669397
],
"per_seed": [
[
0.3939393939393939,
0.46464646464646464,
0.5656565656565656,
0.6767676767676768
],
[
0.3939393939393939,
0.468013468013468,
0.569023569023569,
0.6464646464646465
],
[
0.3939393939393939,
0.4882154882154882,
0.5791245791245792,
0.6565656565656566
],
[
0.3939393939393939,
0.5521885521885522,
0.6161616161616161,
0.6632996632996633
],
[
0.3939393939393939,
0.4713804713804714,
0.6094276094276094,
0.6632996632996633
]
]
},
"no_adabn": {
"k_per_class": [
0,
1,
5,
10
],
"mean": [
0.4006734006734007,
0.45925925925925926,
0.5892255892255892,
0.6734006734006733
],
"std": [
0.0,
0.04219380524665465,
0.01412537169252731,
0.00903461809090825
],
"per_seed": [
[
0.4006734006734007,
0.4377104377104377,
0.5656565656565656,
0.6632996632996633
],
[
0.4006734006734007,
0.41414141414141414,
0.5824915824915825,
0.6632996632996633
],
[
0.4006734006734007,
0.45454545454545453,
0.5993265993265994,
0.6835016835016835
],
[
0.4006734006734007,
0.5387205387205387,
0.5925925925925926,
0.6734006734006734
],
[
0.4006734006734007,
0.4511784511784512,
0.6060606060606061,
0.6835016835016835
]
]
},
"frozen": {
"k_per_class": [
0,
1,
5,
10
],
"mean": [
0.4006734006734007,
0.35757575757575755,
0.5575757575757576,
0.6127946127946128
],
"std": [
0.0,
0.031180705462061222,
0.019470481205119114,
0.02202751142598803
],
"per_seed": [
[
0.4006734006734007,
0.30976430976430974,
0.5521885521885522,
0.6060606060606061
],
[
0.4006734006734007,
0.3400673400673401,
0.5420875420875421,
0.6127946127946128
],
[
0.4006734006734007,
0.3838383838383838,
0.5488215488215489,
0.5757575757575758
],
[
0.4006734006734007,
0.39730639730639733,
0.5488215488215489,
0.6397306397306397
],
[
0.4006734006734007,
0.3569023569023569,
0.5959595959595959,
0.6296296296296297
]
]
},
"random_init": {
"k_per_class": [
0,
1,
5,
10
],
"mean": [
0.09898989898989899,
0.21077441077441078,
0.4673400673400673,
0.6363636363636365
],
"std": [
0.02259658146203706,
0.021986299955979137,
0.02558922558922559,
0.04989530923324634
],
"per_seed": [
[
0.05723905723905724,
0.1919191919191919,
0.4478114478114478,
0.5791245791245792
],
[
0.10101010101010101,
0.24242424242424243,
0.494949494949495,
0.696969696969697
],
[
0.10101010101010101,
0.21212121212121213,
0.4276094276094276,
0.5858585858585859
],
[
0.1111111111111111,
0.2255892255892256,
0.4882154882154882,
0.6902356902356902
],
[
0.12457912457912458,
0.18181818181818182,
0.4781144781144781,
0.6296296296296297
]
]
}
},
"seeds": [
0,
1,
2,
3,
4
],
"failure_examples": {
"setting": "Next Shift, k=10, seed 0",
"n_wrong": 96,
"first": [
{
"target_test_index": 1,
"digit_row": 278,
"true": 2,
"predicted": 0
},
{
"target_test_index": 7,
"digit_row": 1789,
"true": 8,
"predicted": 4
},
{
"target_test_index": 8,
"digit_row": 1571,
"true": 8,
"predicted": 1
},
{
"target_test_index": 10,
"digit_row": 1200,
"true": 7,
"predicted": 4
},
{
"target_test_index": 13,
"digit_row": 550,
"true": 6,
"predicted": 5
},
{
"target_test_index": 17,
"digit_row": 730,
"true": 2,
"predicted": 8
},
{
"target_test_index": 18,
"digit_row": 349,
"true": 1,
"predicted": 8
},
{
"target_test_index": 23,
"digit_row": 1227,
"true": 1,
"predicted": 8
},
{
"target_test_index": 24,
"digit_row": 1790,
"true": 8,
"predicted": 2
},
{
"target_test_index": 25,
"digit_row": 103,
"true": 3,
"predicted": 2
},
{
"target_test_index": 28,
"digit_row": 1199,
"true": 1,
"predicted": 8
},
{
"target_test_index": 30,
"digit_row": 306,
"true": 2,
"predicted": 3
},
{
"target_test_index": 32,
"digit_row": 738,
"true": 9,
"predicted": 1
},
{
"target_test_index": 34,
"digit_row": 141,
"true": 1,
"predicted": 7
},
{
"target_test_index": 35,
"digit_row": 563,
"true": 1,
"predicted": 8
},
{
"target_test_index": 38,
"digit_row": 718,
"true": 0,
"predicted": 4
},
{
"target_test_index": 49,
"digit_row": 1088,
"true": 7,
"predicted": 8
},
{
"target_test_index": 50,
"digit_row": 281,
"true": 5,
"predicted": 9
},
{
"target_test_index": 51,
"digit_row": 447,
"true": 7,
"predicted": 9
},
{
"target_test_index": 54,
"digit_row": 1120,
"true": 1,
"predicted": 8
},
{
"target_test_index": 55,
"digit_row": 579,
"true": 3,
"predicted": 9
},
{
"target_test_index": 62,
"digit_row": 409,
"true": 3,
"predicted": 2
},
{
"target_test_index": 63,
"digit_row": 808,
"true": 5,
"predicted": 3
},
{
"target_test_index": 68,
"digit_row": 765,
"true": 9,
"predicted": 7
},
{
"target_test_index": 77,
"digit_row": 1097,
"true": 1,
"predicted": 8
},
{
"target_test_index": 79,
"digit_row": 225,
"true": 4,
"predicted": 6
},
{
"target_test_index": 82,
"digit_row": 930,
"true": 5,
"predicted": 9
},
{
"target_test_index": 93,
"digit_row": 121,
"true": 4,
"predicted": 1
},
{
"target_test_index": 95,
"digit_row": 1247,
"true": 1,
"predicted": 7
},
{
"target_test_index": 97,
"digit_row": 104,
"true": 6,
"predicted": 1
}
]
},
"tests": {
"reload": {
"returncode": 0,
"seconds": 0.25,
"process": "fresh python -E subprocess, cwd = temporary copy of the model folder",
"passed": true,
"max_abs_diff": 0.0,
"n_images": 20,
"n_labels": 10
},
"node_parity": {
"cases": 3,
"failed": 0,
"maxDiff": 5.412337245047638e-16,
"results": [
{
"name": "zero",
"maxDiff": 3.3306690738754696e-16
},
{
"name": "one-each",
"maxDiff": 5.412337245047638e-16
},
{
"name": "twenty",
"maxDiff": 4.440892098500626e-16
}
],
"ms": 2798,
"runtime": "node v22.23.4",
"status": "passed"
},
"browser": {
"status": "skipped",
"reason": "playwright not installed"
}
}
}