enzoferracini's picture
feat(model): publish selected local mDeBERTa research pipeline
f12d393 verified
Raw History Blame Contribute Delete
12.8 kB
{
"selected": {
"macro_f1": 0.9458477407255466,
"f1": 0.9380530973451328,
"average_precision": 0.978941878575674,
"folds": 3,
"fold_reports": [
{
"train_groups": 26,
"validation_groups": 13,
"macro_f1": 0.9793303017775941,
"average_precision": 0.997835497835498
},
{
"train_groups": 26,
"validation_groups": 13,
"macro_f1": 0.9083333333333333,
"average_precision": 0.9552032125875916
},
{
"train_groups": 26,
"validation_groups": 13,
"macro_f1": 0.9460227272727273,
"average_precision": 0.9938080495356038
}
],
"oof_scores": [
0.9974944591522217,
0.9955982565879822,
0.9964062571525574,
0.001573236775584519,
0.0018356473883613944,
0.0005117384134791791,
0.0012508711079135537,
0.0028142575174570084,
0.921700656414032,
0.8017196655273438,
0.891438364982605,
0.0019709926564246416,
0.0009275616612285376,
8.962978608906269e-05,
0.0011396443005651236,
0.0010641987901180983,
0.9876701831817627,
0.02176378294825554,
0.9956931471824646,
0.0023894296027719975,
0.02176770567893982,
0.0005863080150447786,
0.0064889066852629185,
0.007131799589842558,
0.8945651054382324,
0.08026428520679474,
0.9969968795776367,
0.04250901937484741,
0.005009626969695091,
0.00011741025082301348,
0.17814192175865173,
0.01390241552144289,
0.9233620762825012,
0.8129627108573914,
0.9954577684402466,
0.0012587251840159297,
0.0008041538530960679,
0.00038639537524431944,
0.0023932475596666336,
0.0007668251055292785,
0.9966105818748474,
0.9917599558830261,
0.9957504272460938,
0.009498599916696548,
0.18038657307624817,
0.00010709496564231813,
0.9192822575569153,
0.5587064623832703,
0.9924682974815369,
0.9975998997688293,
0.9975301623344421,
0.008086169138550758,
0.00856433343142271,
0.0003519972087815404,
0.9869614839553833,
0.010971194133162498,
0.9844051003456116,
0.9924294352531433,
0.9976155757904053,
0.0031317120883613825,
0.01994358003139496,
0.000114981405204162,
0.004176675342023373,
0.0015110494568943977,
0.8079658150672913,
0.8175349235534668,
0.9531922936439514,
0.0011087239254266024,
0.0028624178376048803,
0.00013224428403191268,
0.001569352112710476,
0.0024940904695540667,
0.6651837229728699,
0.0006137837190181017,
0.9904462099075317,
0.0004961069207638502,
0.9604456424713135,
0.041432347148656845,
0.9967343807220459,
0.0026492674369364977,
0.9796324372291565,
0.0006179233314469457,
0.9972860813140869,
0.033348143100738525,
0.9975593090057373,
0.004417634569108486,
0.9963451027870178,
0.0008816909394226968,
0.9687778949737549,
0.0007139152730815113,
0.4262998700141907,
0.003795438213273883,
0.9909142255783081,
0.0010800814488902688,
0.967029869556427,
0.0029997387900948524,
0.997118353843689,
0.0007074209279380739,
0.9870109558105469,
0.0014369282871484756,
0.9849678874015808,
0.0028716952074319124,
0.9966367483139038,
0.0006909564253874123,
0.9940197467803955,
0.007613298948854208,
0.9974876642227173,
0.0015202444046735764,
0.9335872530937195,
0.0007145786075852811,
0.9187936186790466,
0.275179922580719,
0.9900261163711548,
0.0014856801135465503,
0.9868248105049133,
0.013191863894462585,
0.8087249398231506,
0.017571013420820236,
0.7537350058555603,
0.001877651084214449,
0.984703779220581,
0.001089011668227613,
0.9897966980934143,
0.001932453247718513,
0.9445369243621826,
0.0040660700760781765,
0.99272221326828,
0.0022476951126009226,
0.038174573332071304,
0.0005725350929424167,
0.966562032699585,
0.0018391491612419486
],
"raw_oof_scores": [
0.9974944591522217,
0.9955982565879822,
0.9964062571525574,
0.001573236775584519,
0.0018356473883613944,
0.0005117384134791791,
0.0012508711079135537,
0.0028142575174570084,
0.921700656414032,
0.8017196655273438,
0.891438364982605,
0.0019709926564246416,
0.0009275616612285376,
8.962978608906269e-05,
0.0011396443005651236,
0.0010641987901180983,
0.9876701831817627,
0.02176378294825554,
0.9956931471824646,
0.0023894296027719975,
0.02176770567893982,
0.0005863080150447786,
0.0064889066852629185,
0.007131799589842558,
0.8945651054382324,
0.08026428520679474,
0.9969968795776367,
0.04250901937484741,
0.005009626969695091,
0.00011741025082301348,
0.17814192175865173,
0.01390241552144289,
0.9233620762825012,
0.8129627108573914,
0.9954577684402466,
0.0012587251840159297,
0.0008041538530960679,
0.00038639537524431944,
0.0023932475596666336,
0.0007668251055292785,
0.9966105818748474,
0.9917599558830261,
0.9957504272460938,
0.009498599916696548,
0.18038657307624817,
0.00010709496564231813,
0.9192822575569153,
0.5587064623832703,
0.9924682974815369,
0.9975998997688293,
0.9975301623344421,
0.008086169138550758,
0.00856433343142271,
0.0003519972087815404,
0.9869614839553833,
0.010971194133162498,
0.9844051003456116,
0.9924294352531433,
0.9976155757904053,
0.0031317120883613825,
0.01994358003139496,
0.000114981405204162,
0.004176675342023373,
0.0015110494568943977,
0.8079658150672913,
0.8175349235534668,
0.9531922936439514,
0.0011087239254266024,
0.0028624178376048803,
0.00013224428403191268,
0.001569352112710476,
0.0024940904695540667,
0.6651837229728699,
0.0006137837190181017,
0.9904462099075317,
0.0004961069207638502,
0.9604456424713135,
0.041432347148656845,
0.9967343807220459,
0.0026492674369364977,
0.9796324372291565,
0.0006179233314469457,
0.9972860813140869,
0.033348143100738525,
0.9975593090057373,
0.004417634569108486,
0.9963451027870178,
0.0008816909394226968,
0.9687778949737549,
0.0007139152730815113,
0.4262998700141907,
0.003795438213273883,
0.9909142255783081,
0.0010800814488902688,
0.967029869556427,
0.0029997387900948524,
0.997118353843689,
0.0007074209279380739,
0.9870109558105469,
0.0014369282871484756,
0.9849678874015808,
0.0028716952074319124,
0.9966367483139038,
0.0006909564253874123,
0.9940197467803955,
0.007613298948854208,
0.9974876642227173,
0.0015202444046735764,
0.9335872530937195,
0.0007145786075852811,
0.9187936186790466,
0.275179922580719,
0.9900261163711548,
0.0014856801135465503,
0.9868248105049133,
0.013191863894462585,
0.8087249398231506,
0.017571013420820236,
0.7537350058555603,
0.001877651084214449,
0.984703779220581,
0.001089011668227613,
0.9897966980934143,
0.001932453247718513,
0.9445369243621826,
0.0040660700760781765,
0.99272221326828,
0.0022476951126009226,
0.038174573332071304,
0.0005725350929424167,
0.966562032699585,
0.0018391491612419486
],
"oof_binary": [
1,
1,
1,
0,
0,
0,
0,
0,
1,
1,
1,
0,
0,
0,
0,
0,
1,
0,
1,
0,
0,
0,
0,
0,
1,
0,
1,
0,
0,
0,
0,
0,
1,
1,
1,
0,
0,
0,
0,
0,
1,
1,
1,
0,
0,
0,
1,
1,
1,
1,
1,
0,
0,
0,
1,
0,
1,
1,
1,
0,
0,
0,
0,
0,
1,
1,
1,
0,
0,
0,
0,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
0,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
1,
0,
0,
0,
1,
0
],
"template": {
"kind": "average",
"indices": [
15
],
"weights": [
1.0
],
"calibrator_kind": "identity"
}
},
"components": [
{
"name": "hpo_5::min",
"directory": "artifacts/v03/hpo_5/model",
"initial_key": "nli_deberta",
"aggregation": "min",
"quantized": false,
"model_sha256": "058be8f2224e2f995de5fd3c7b4b14bc3df19abc7874be82793727af2c1b68bf"
}
],
"graphs_evaluated": 210,
"development_pairs": 132,
"development_groups": 39,
"config": {
"seed": 42,
"training_source": "original train plus within-scenario counterfactual combinations",
"training_pairs": 744,
"development": [
"data/v01/validation.jsonl",
"data/v03/development.jsonl"
],
"calibration": "data/v03/calibration.jsonl",
"test": "data/v03/test.jsonl",
"folds": 3,
"selection": "cross-fitted macro F1, then average precision; earliest tie",
"nli_models": {
"nli_minilm": {
"directory": "artifacts/nli-minilm",
"repo": "MoritzLaurer/multilingual-MiniLMv2-L6-mnli-xnli",
"revision": "0a71e92a985b6e1ad1828cf67ce9c459639c1dca",
"license": "mit"
},
"nli_deberta": {
"directory": "artifacts/nli-deberta",
"repo": "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli",
"revision": "8adb042d524ecd5c26d3e3ba0e3fbcf7e2d0864c",
"license": "mit"
}
},
"default_training": {
"epochs": 3,
"batch_size": 8,
"gradient_accumulation": 2,
"learning_rate": 1e-05,
"freeze_embeddings": true,
"warmup_ratio": 0.1,
"weight_decay": 0.01,
"label_smoothing": 0.05,
"patience": 2,
"bf16": true,
"layer_lr_decay": 0.95
},
"hpo_trials": 6,
"hpo_max_epochs": 2,
"hpo_sampler": "TPE with two random startup trials; Hyperband pruning",
"large_finetune": {
"epochs": 4,
"batch_size": 4,
"gradient_accumulation": 4,
"learning_rate": 5e-06,
"freeze_lower_layers": 6,
"ema_decay": 0.98,
"rdrop": 0.05,
"gradient_checkpointing": true
},
"lora": {
"epochs": 3,
"batch_size": 4,
"gradient_accumulation": 4,
"learning_rate": 3e-05,
"lora": true,
"gradient_checkpointing": true
},
"distillation": {
"epochs": 3,
"distillation_weight": 0.3,
"temperature": 2,
"hard_sampling": true,
"rdrop": 0.1,
"ema_decay": 0.98
},
"additional_seed": 137,
"domain_adaptation_steps": 40,
"calibrators": [
"identity",
"sigmoid",
"isotonic"
],
"ensembles": [
"equal averaging",
"nonnegative weighted averaging",
"stacking"
],
"ensemble_weight_trials": 48,
"conformal_alpha": 0.1,
"cpu_quantization": "dynamic int8 evaluated as a development candidate",
"test_policy": "Open once only after scoring graph and calibration are frozen; old tests remain diagnostic",
"limitations": [
"Same AI author and reviewer; no human review",
"Calibration only twenty pairs",
"Cross-fitting is for selection, not an independent test"
],
"hpo_backbones": [
"nli_minilm",
"nli_deberta"
],
"preferred_backbone": "nli_deberta"
},
"model_epoch_selection_used_development": true,
"limitation": "Cross-fitting controls postprocessor fitting, not all adaptive model selection bias",
"graph_sha256": "d3a4e63c7112159551d89af51e89c8d17254a0d5b976c430da1879fb6477b92d",
"test_opened": false,
"calibration_opened": false
}