{ "chosen_finetuned": "online_contrastive", "chosen_bi_encoder": "online_contrastive", "candidates": [ { "name": "lexical", "directory": "", "kind": "lexical", "policy": { "binary_threshold": 0.34169672598972345, "thresholds": { "low": 0.34169672598972345, "high": 1.0, "positive_enabled": false, "negative_enabled": true }, "selection_split": "validation", "precision_target": 0.9, "negative_predictive_value_target": 0.9, "minimum_support": 5, "decision_policy_version": "0.1.0" }, "validation": { "support": 72, "positive_support": 27, "precision": 0.43333333333333335, "recall": 0.9629629629629629, "f1": 0.5977011494252874, "macro_f1": 0.49183303085299457, "average_precision": 0.3708027821398918, "confusion_matrix": [ [ 11, 34 ], [ 1, 26 ] ], "coverage": 0.16666666666666666, "review_rate": 0.8333333333333334, "review_count": 60, "redundant_count": 0, "not_redundant_count": 12, "selective_precision": null, "negative_predictive_value": 0.9166666666666666 } }, { "name": "base", "directory": "artifacts/base", "kind": "cosine", "policy": { "binary_threshold": 0.7951657772064209, "thresholds": { "low": 0.6722568273544312, "high": 1.0, "positive_enabled": false, "negative_enabled": true }, "selection_split": "validation", "precision_target": 0.9, "negative_predictive_value_target": 0.9, "minimum_support": 5, "decision_policy_version": "0.1.0" }, "validation": { "support": 72, "positive_support": 27, "precision": 0.5882352941176471, "recall": 0.7407407407407407, "f1": 0.6557377049180327, "macro_f1": 0.7013628283626308, "average_precision": 0.6631701186120071, "confusion_matrix": [ [ 31, 14 ], [ 7, 20 ] ], "coverage": 0.19444444444444445, "review_rate": 0.8055555555555556, "review_count": 58, "redundant_count": 0, "not_redundant_count": 14, "selective_precision": null, "negative_predictive_value": 0.9285714285714286 } }, { "name": "incumbent", "directory": "artifacts/run-42/selected-model", "kind": "cosine", "policy": { "binary_threshold": 0.47051626443862915, "thresholds": { "low": 0.5057744979858398, "high": 0.8676812648773193, "positive_enabled": true, "negative_enabled": true }, "selection_split": "validation", "precision_target": 0.9, "negative_predictive_value_target": 0.9, "minimum_support": 5, "decision_policy_version": "0.1.0" }, "validation": { "support": 72, "positive_support": 27, "precision": 0.5192307692307693, "recall": 1.0, "f1": 0.6835443037974683, "macro_f1": 0.6494644595910419, "average_precision": 0.7229152368252679, "confusion_matrix": [ [ 20, 25 ], [ 0, 27 ] ], "coverage": 0.4305555555555556, "review_rate": 0.5694444444444444, "review_count": 41, "redundant_count": 8, "not_redundant_count": 23, "selective_precision": 1.0, "negative_predictive_value": 0.9130434782608695 } }, { "name": "contrastive", "directory": "artifacts/extended-42/contrastive/model", "kind": "cosine", "policy": { "binary_threshold": 0.5507670044898987, "thresholds": { "low": 0.5904079675674438, "high": 0.8942374587059021, "positive_enabled": true, "negative_enabled": true }, "selection_split": "validation", "precision_target": 0.9, "negative_predictive_value_target": 0.9, "minimum_support": 5, "decision_policy_version": "0.1.0" }, "validation": { "support": 72, "positive_support": 27, "precision": 0.54, "recall": 1.0, "f1": 0.7012987012987013, "macro_f1": 0.6790075596045746, "average_precision": 0.7632222551191008, "confusion_matrix": [ [ 22, 23 ], [ 0, 27 ] ], "coverage": 0.5, "review_rate": 0.5, "review_count": 36, "redundant_count": 10, "not_redundant_count": 26, "selective_precision": 0.9, "negative_predictive_value": 0.9230769230769231 } }, { "name": "online_contrastive", "directory": "artifacts/extended-42/online_contrastive/model", "kind": "cosine", "policy": { "binary_threshold": 0.7588188052177429, "thresholds": { "low": 0.6037070751190186, "high": 0.8967244625091553, "positive_enabled": true, "negative_enabled": true }, "selection_split": "validation", "precision_target": 0.9, "negative_predictive_value_target": 0.9, "minimum_support": 5, "decision_policy_version": "0.1.0" }, "validation": { "support": 72, "positive_support": 27, "precision": 0.6774193548387096, "recall": 0.7777777777777778, "f1": 0.7241379310344828, "macro_f1": 0.7690457097032879, "average_precision": 0.7662009908295464, "confusion_matrix": [ [ 35, 10 ], [ 6, 21 ] ], "coverage": 0.5, "review_rate": 0.5, "review_count": 36, "redundant_count": 11, "not_redundant_count": 25, "selective_precision": 0.9090909090909091, "negative_predictive_value": 0.92 } }, { "name": "cosent_balanced", "directory": "artifacts/extended-42/cosent_balanced/model", "kind": "cosine", "policy": { "binary_threshold": 0.6639197468757629, "thresholds": { "low": 0.4964098632335663, "high": 0.8445258140563965, "positive_enabled": true, "negative_enabled": true }, "selection_split": "validation", "precision_target": 0.9, "negative_predictive_value_target": 0.9, "minimum_support": 5, "decision_policy_version": "0.1.0" }, "validation": { "support": 72, "positive_support": 27, "precision": 0.5945945945945946, "recall": 0.8148148148148148, "f1": 0.6875, "macro_f1": 0.71875, "average_precision": 0.7527715859363884, "confusion_matrix": [ [ 30, 15 ], [ 5, 22 ] ], "coverage": 0.4722222222222222, "review_rate": 0.5277777777777778, "review_count": 38, "redundant_count": 10, "not_redundant_count": 24, "selective_precision": 0.9, "negative_predictive_value": 0.9166666666666666 } }, { "name": "cosent_frozen", "directory": "artifacts/extended-42/cosent_frozen/model", "kind": "cosine", "policy": { "binary_threshold": 0.47448647022247314, "thresholds": { "low": 0.4975001811981201, "high": 0.8474057912826538, "positive_enabled": true, "negative_enabled": true }, "selection_split": "validation", "precision_target": 0.9, "negative_predictive_value_target": 0.9, "minimum_support": 5, "decision_policy_version": "0.1.0" }, "validation": { "support": 72, "positive_support": 27, "precision": 0.5294117647058824, "recall": 1.0, "f1": 0.6923076923076923, "macro_f1": 0.6643356643356644, "average_precision": 0.7605507017018385, "confusion_matrix": [ [ 21, 24 ], [ 0, 27 ] ], "coverage": 0.5, "review_rate": 0.5, "review_count": 36, "redundant_count": 11, "not_redundant_count": 25, "selective_precision": 0.9090909090909091, "negative_predictive_value": 0.92 } }, { "name": "cross_encoder", "directory": "artifacts/extended-42/cross_encoder/model", "kind": "cross_encoder", "policy": { "binary_threshold": 0.40319615602493286, "thresholds": { "low": 0.20058020949363708, "high": 1.0, "positive_enabled": false, "negative_enabled": true }, "selection_split": "validation", "precision_target": 0.9, "negative_predictive_value_target": 0.9, "minimum_support": 5, "decision_policy_version": "0.1.0" }, "validation": { "support": 72, "positive_support": 27, "precision": 0.6875, "recall": 0.8148148148148148, "f1": 0.7457627118644068, "macro_f1": 0.7846460618145563, "average_precision": 0.7011072722166704, "confusion_matrix": [ [ 35, 10 ], [ 5, 22 ] ], "coverage": 0.4444444444444444, "review_rate": 0.5555555555555556, "review_count": 40, "redundant_count": 0, "not_redundant_count": 32, "selective_precision": null, "negative_predictive_value": 0.90625 } }, { "name": "cross_encoder_frozen", "directory": "artifacts/extended-42/cross_encoder_frozen/model", "kind": "cross_encoder", "policy": { "binary_threshold": 0.43186578154563904, "thresholds": { "low": 0.15257886052131653, "high": 1.0, "positive_enabled": false, "negative_enabled": true }, "selection_split": "validation", "precision_target": 0.9, "negative_predictive_value_target": 0.9, "minimum_support": 5, "decision_policy_version": "0.1.0" }, "validation": { "support": 72, "positive_support": 27, "precision": 0.6571428571428571, "recall": 0.8518518518518519, "f1": 0.7419354838709677, "macro_f1": 0.7734067663257278, "average_precision": 0.6770074245823222, "confusion_matrix": [ [ 33, 12 ], [ 4, 23 ] ], "coverage": 0.2916666666666667, "review_rate": 0.7083333333333334, "review_count": 51, "redundant_count": 0, "not_redundant_count": 21, "selective_precision": null, "negative_predictive_value": 0.9047619047619048 } } ], "skipped": [ { "technique": "pair feature classifier", "reason": "A tool write was blocked; feature baseline was not implemented or evaluated" } ], "config": { "base_revision": "e8f8c211226b894fcb81acc59f3b34ba3efd5f42", "base_directory": "artifacts/base", "initial_directory": "artifacts/run-42/selected-model", "dataset": "data/v01", "refinement_dataset": "data/refinement", "fresh_holdout": "data/v02", "seed": 42, "batch_size": 8, "gradient_accumulation": 2, "epochs": 3, "learning_rate": 1e-05, "loss": "cosent", "selection": "validation average_precision, then binary F1; earliest tie", "bi_candidates": [ { "name": "contrastive", "loss": "contrastive", "margin": 0.5 }, { "name": "online_contrastive", "loss": "online_contrastive", "margin": 0.5, "batch_size": 16 }, { "name": "cosent_balanced", "loss": "cosent", "learning_rate": 5e-06, "sampling": "balanced" }, { "name": "cosent_frozen", "loss": "cosent", "epochs": 4, "freeze_embeddings": true, "freeze_lower_layers": 6 } ], "cross_candidates": [ { "name": "cross_encoder", "epochs": 6, "learning_rate": 2e-05 }, { "name": "cross_encoder_frozen", "epochs": 6, "learning_rate": 3e-05, "freeze_embeddings": true, "freeze_lower_layers": 6 } ], "pair_feature_logistic_C": [ 1.0, 10.0 ], "test_policy": "Fresh holdout scored once after all v02 selection; previous tests are diagnostics only" }, "fresh_holdout": { "groups": 24, "pairs": 48, "sha256": { "test.jsonl": "59af8f19c4c157a32b818ee021d34f02fa6178077baf577a37e3c89b603d9931" }, "source_sha256": "e9f04b8b9385f2906eb56587e9a822ee344a4a162f94b9d2e9a7237dfb23fa95", "provenance": { "author": "same authoring AI", "human_reviewed": false, "license": "CC-BY-4.0", "purpose": "Fresh scenarios frozen before v02 selection; old tests are diagnostic only", "limitation": "General design informed by earlier aggregate errors; not independent human authorship" } }, "dataset": { "schema_version": "0.1.0", "seed": 42, "groups": 60, "pairs": 480, "groups_by_split": { "train": 42, "test": 9, "validation": 9 }, "source_sha256": "4539d298a41c756779270c7ab5f69a179249d9dc89e711fd1c530aa37549c6b1", "sha256": { "train.jsonl": "f1a038ea8e5acd0319d15100e8574aa95e1638190ad81adf58a89fa821150d8e", "validation.jsonl": "12083d803d6c13a295c63eaa8bb311e7a0fa07e407374bb13af1567877166696", "test.jsonl": "c857897c9a5b3a72d33927540810a625f5bc4c2ab8f7ea2abf55d54933d71bbd", "all.jsonl": "32c4154fb356a73d28fadd52339a619f5efc288c024d5dadc437ee4d680ff882" }, "provenance": { "source": "scenarios.psv", "author": "Codex authoring AI", "reviewer": "same authoring AI; no independent or human review", "human_reviewed": false, "license": "CC-BY-4.0", "limitation": "Shared authored constructions; not representative of real CVs" } }, "refinement_dataset": { "schema_version": "0.1.0", "seed": 42, "groups": 20, "pairs": 160, "groups_by_split": { "train": 20 }, "source_sha256": "cbf531c96059f20f5657de32435272623e8de46e1d08daadaeae206ebcdad2ad", "sha256": { "train.jsonl": "6b3cb804cf877c500aa567b0e583afe9dc684cd5c9fe567cb25593613e28831a" }, "provenance": { "source": "refinement.psv", "author": "Codex authoring AI", "reviewer": "same authoring AI; no independent or human review", "human_reviewed": false, "license": "CC-BY-4.0", "limitation": "Shared authored constructions; not representative of real CVs" } }, "test_evaluated": false, "selection_rule": "validation average_precision, then binary F1; earliest tie" }