{ "selected": { "macro_f1": 0.9458477407255466, "f1": 0.9380530973451328, "average_precision": 0.978941878575674, "folds": 3, "fold_reports": [ { "train_groups": 26, "validation_groups": 13, "macro_f1": 0.9793303017775941, "average_precision": 0.997835497835498 }, { "train_groups": 26, "validation_groups": 13, "macro_f1": 0.9083333333333333, "average_precision": 0.9552032125875916 }, { "train_groups": 26, "validation_groups": 13, "macro_f1": 0.9460227272727273, "average_precision": 0.9938080495356038 } ], "oof_scores": [ 0.9974944591522217, 0.9955982565879822, 0.9964062571525574, 0.001573236775584519, 0.0018356473883613944, 0.0005117384134791791, 0.0012508711079135537, 0.0028142575174570084, 0.921700656414032, 0.8017196655273438, 0.891438364982605, 0.0019709926564246416, 0.0009275616612285376, 8.962978608906269e-05, 0.0011396443005651236, 0.0010641987901180983, 0.9876701831817627, 0.02176378294825554, 0.9956931471824646, 0.0023894296027719975, 0.02176770567893982, 0.0005863080150447786, 0.0064889066852629185, 0.007131799589842558, 0.8945651054382324, 0.08026428520679474, 0.9969968795776367, 0.04250901937484741, 0.005009626969695091, 0.00011741025082301348, 0.17814192175865173, 0.01390241552144289, 0.9233620762825012, 0.8129627108573914, 0.9954577684402466, 0.0012587251840159297, 0.0008041538530960679, 0.00038639537524431944, 0.0023932475596666336, 0.0007668251055292785, 0.9966105818748474, 0.9917599558830261, 0.9957504272460938, 0.009498599916696548, 0.18038657307624817, 0.00010709496564231813, 0.9192822575569153, 0.5587064623832703, 0.9924682974815369, 0.9975998997688293, 0.9975301623344421, 0.008086169138550758, 0.00856433343142271, 0.0003519972087815404, 0.9869614839553833, 0.010971194133162498, 0.9844051003456116, 0.9924294352531433, 0.9976155757904053, 0.0031317120883613825, 0.01994358003139496, 0.000114981405204162, 0.004176675342023373, 0.0015110494568943977, 0.8079658150672913, 0.8175349235534668, 0.9531922936439514, 0.0011087239254266024, 0.0028624178376048803, 0.00013224428403191268, 0.001569352112710476, 0.0024940904695540667, 0.6651837229728699, 0.0006137837190181017, 0.9904462099075317, 0.0004961069207638502, 0.9604456424713135, 0.041432347148656845, 0.9967343807220459, 0.0026492674369364977, 0.9796324372291565, 0.0006179233314469457, 0.9972860813140869, 0.033348143100738525, 0.9975593090057373, 0.004417634569108486, 0.9963451027870178, 0.0008816909394226968, 0.9687778949737549, 0.0007139152730815113, 0.4262998700141907, 0.003795438213273883, 0.9909142255783081, 0.0010800814488902688, 0.967029869556427, 0.0029997387900948524, 0.997118353843689, 0.0007074209279380739, 0.9870109558105469, 0.0014369282871484756, 0.9849678874015808, 0.0028716952074319124, 0.9966367483139038, 0.0006909564253874123, 0.9940197467803955, 0.007613298948854208, 0.9974876642227173, 0.0015202444046735764, 0.9335872530937195, 0.0007145786075852811, 0.9187936186790466, 0.275179922580719, 0.9900261163711548, 0.0014856801135465503, 0.9868248105049133, 0.013191863894462585, 0.8087249398231506, 0.017571013420820236, 0.7537350058555603, 0.001877651084214449, 0.984703779220581, 0.001089011668227613, 0.9897966980934143, 0.001932453247718513, 0.9445369243621826, 0.0040660700760781765, 0.99272221326828, 0.0022476951126009226, 0.038174573332071304, 0.0005725350929424167, 0.966562032699585, 0.0018391491612419486 ], "raw_oof_scores": [ 0.9974944591522217, 0.9955982565879822, 0.9964062571525574, 0.001573236775584519, 0.0018356473883613944, 0.0005117384134791791, 0.0012508711079135537, 0.0028142575174570084, 0.921700656414032, 0.8017196655273438, 0.891438364982605, 0.0019709926564246416, 0.0009275616612285376, 8.962978608906269e-05, 0.0011396443005651236, 0.0010641987901180983, 0.9876701831817627, 0.02176378294825554, 0.9956931471824646, 0.0023894296027719975, 0.02176770567893982, 0.0005863080150447786, 0.0064889066852629185, 0.007131799589842558, 0.8945651054382324, 0.08026428520679474, 0.9969968795776367, 0.04250901937484741, 0.005009626969695091, 0.00011741025082301348, 0.17814192175865173, 0.01390241552144289, 0.9233620762825012, 0.8129627108573914, 0.9954577684402466, 0.0012587251840159297, 0.0008041538530960679, 0.00038639537524431944, 0.0023932475596666336, 0.0007668251055292785, 0.9966105818748474, 0.9917599558830261, 0.9957504272460938, 0.009498599916696548, 0.18038657307624817, 0.00010709496564231813, 0.9192822575569153, 0.5587064623832703, 0.9924682974815369, 0.9975998997688293, 0.9975301623344421, 0.008086169138550758, 0.00856433343142271, 0.0003519972087815404, 0.9869614839553833, 0.010971194133162498, 0.9844051003456116, 0.9924294352531433, 0.9976155757904053, 0.0031317120883613825, 0.01994358003139496, 0.000114981405204162, 0.004176675342023373, 0.0015110494568943977, 0.8079658150672913, 0.8175349235534668, 0.9531922936439514, 0.0011087239254266024, 0.0028624178376048803, 0.00013224428403191268, 0.001569352112710476, 0.0024940904695540667, 0.6651837229728699, 0.0006137837190181017, 0.9904462099075317, 0.0004961069207638502, 0.9604456424713135, 0.041432347148656845, 0.9967343807220459, 0.0026492674369364977, 0.9796324372291565, 0.0006179233314469457, 0.9972860813140869, 0.033348143100738525, 0.9975593090057373, 0.004417634569108486, 0.9963451027870178, 0.0008816909394226968, 0.9687778949737549, 0.0007139152730815113, 0.4262998700141907, 0.003795438213273883, 0.9909142255783081, 0.0010800814488902688, 0.967029869556427, 0.0029997387900948524, 0.997118353843689, 0.0007074209279380739, 0.9870109558105469, 0.0014369282871484756, 0.9849678874015808, 0.0028716952074319124, 0.9966367483139038, 0.0006909564253874123, 0.9940197467803955, 0.007613298948854208, 0.9974876642227173, 0.0015202444046735764, 0.9335872530937195, 0.0007145786075852811, 0.9187936186790466, 0.275179922580719, 0.9900261163711548, 0.0014856801135465503, 0.9868248105049133, 0.013191863894462585, 0.8087249398231506, 0.017571013420820236, 0.7537350058555603, 0.001877651084214449, 0.984703779220581, 0.001089011668227613, 0.9897966980934143, 0.001932453247718513, 0.9445369243621826, 0.0040660700760781765, 0.99272221326828, 0.0022476951126009226, 0.038174573332071304, 0.0005725350929424167, 0.966562032699585, 0.0018391491612419486 ], "oof_binary": [ 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0 ], "template": { "kind": "average", "indices": [ 15 ], "weights": [ 1.0 ], "calibrator_kind": "identity" } }, "components": [ { "name": "hpo_5::min", "directory": "artifacts/v03/hpo_5/model", "initial_key": "nli_deberta", "aggregation": "min", "quantized": false, "model_sha256": "058be8f2224e2f995de5fd3c7b4b14bc3df19abc7874be82793727af2c1b68bf" } ], "graphs_evaluated": 210, "development_pairs": 132, "development_groups": 39, "config": { "seed": 42, "training_source": "original train plus within-scenario counterfactual combinations", "training_pairs": 744, "development": [ "data/v01/validation.jsonl", "data/v03/development.jsonl" ], "calibration": "data/v03/calibration.jsonl", "test": "data/v03/test.jsonl", "folds": 3, "selection": "cross-fitted macro F1, then average precision; earliest tie", "nli_models": { "nli_minilm": { "directory": "artifacts/nli-minilm", "repo": "MoritzLaurer/multilingual-MiniLMv2-L6-mnli-xnli", "revision": "0a71e92a985b6e1ad1828cf67ce9c459639c1dca", "license": "mit" }, "nli_deberta": { "directory": "artifacts/nli-deberta", "repo": "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", "revision": "8adb042d524ecd5c26d3e3ba0e3fbcf7e2d0864c", "license": "mit" } }, "default_training": { "epochs": 3, "batch_size": 8, "gradient_accumulation": 2, "learning_rate": 1e-05, "freeze_embeddings": true, "warmup_ratio": 0.1, "weight_decay": 0.01, "label_smoothing": 0.05, "patience": 2, "bf16": true, "layer_lr_decay": 0.95 }, "hpo_trials": 6, "hpo_max_epochs": 2, "hpo_sampler": "TPE with two random startup trials; Hyperband pruning", "large_finetune": { "epochs": 4, "batch_size": 4, "gradient_accumulation": 4, "learning_rate": 5e-06, "freeze_lower_layers": 6, "ema_decay": 0.98, "rdrop": 0.05, "gradient_checkpointing": true }, "lora": { "epochs": 3, "batch_size": 4, "gradient_accumulation": 4, "learning_rate": 3e-05, "lora": true, "gradient_checkpointing": true }, "distillation": { "epochs": 3, "distillation_weight": 0.3, "temperature": 2, "hard_sampling": true, "rdrop": 0.1, "ema_decay": 0.98 }, "additional_seed": 137, "domain_adaptation_steps": 40, "calibrators": [ "identity", "sigmoid", "isotonic" ], "ensembles": [ "equal averaging", "nonnegative weighted averaging", "stacking" ], "ensemble_weight_trials": 48, "conformal_alpha": 0.1, "cpu_quantization": "dynamic int8 evaluated as a development candidate", "test_policy": "Open once only after scoring graph and calibration are frozen; old tests remain diagnostic", "limitations": [ "Same AI author and reviewer; no human review", "Calibration only twenty pairs", "Cross-fitting is for selection, not an independent test" ], "hpo_backbones": [ "nli_minilm", "nli_deberta" ], "preferred_backbone": "nli_deberta" }, "model_epoch_selection_used_development": true, "limitation": "Cross-fitting controls postprocessor fitting, not all adaptive model selection bias", "graph_sha256": "d3a4e63c7112159551d89af51e89c8d17254a0d5b976c430da1879fb6477b92d", "test_opened": false, "calibration_opened": false }