enzoferracini's picture
feat(model): publish trained experimental online contrastive encoder
f0b6377 verified
Raw History Blame Contribute Delete
15.5 kB
{
"chosen_finetuned": "online_contrastive",
"chosen_bi_encoder": "online_contrastive",
"candidates": [
{
"name": "lexical",
"directory": "",
"kind": "lexical",
"policy": {
"binary_threshold": 0.34169672598972345,
"thresholds": {
"low": 0.34169672598972345,
"high": 1.0,
"positive_enabled": false,
"negative_enabled": true
},
"selection_split": "validation",
"precision_target": 0.9,
"negative_predictive_value_target": 0.9,
"minimum_support": 5,
"decision_policy_version": "0.1.0"
},
"validation": {
"support": 72,
"positive_support": 27,
"precision": 0.43333333333333335,
"recall": 0.9629629629629629,
"f1": 0.5977011494252874,
"macro_f1": 0.49183303085299457,
"average_precision": 0.3708027821398918,
"confusion_matrix": [
[
11,
34
],
[
1,
26
]
],
"coverage": 0.16666666666666666,
"review_rate": 0.8333333333333334,
"review_count": 60,
"redundant_count": 0,
"not_redundant_count": 12,
"selective_precision": null,
"negative_predictive_value": 0.9166666666666666
}
},
{
"name": "base",
"directory": "artifacts/base",
"kind": "cosine",
"policy": {
"binary_threshold": 0.7951657772064209,
"thresholds": {
"low": 0.6722568273544312,
"high": 1.0,
"positive_enabled": false,
"negative_enabled": true
},
"selection_split": "validation",
"precision_target": 0.9,
"negative_predictive_value_target": 0.9,
"minimum_support": 5,
"decision_policy_version": "0.1.0"
},
"validation": {
"support": 72,
"positive_support": 27,
"precision": 0.5882352941176471,
"recall": 0.7407407407407407,
"f1": 0.6557377049180327,
"macro_f1": 0.7013628283626308,
"average_precision": 0.6631701186120071,
"confusion_matrix": [
[
31,
14
],
[
7,
20
]
],
"coverage": 0.19444444444444445,
"review_rate": 0.8055555555555556,
"review_count": 58,
"redundant_count": 0,
"not_redundant_count": 14,
"selective_precision": null,
"negative_predictive_value": 0.9285714285714286
}
},
{
"name": "incumbent",
"directory": "artifacts/run-42/selected-model",
"kind": "cosine",
"policy": {
"binary_threshold": 0.47051626443862915,
"thresholds": {
"low": 0.5057744979858398,
"high": 0.8676812648773193,
"positive_enabled": true,
"negative_enabled": true
},
"selection_split": "validation",
"precision_target": 0.9,
"negative_predictive_value_target": 0.9,
"minimum_support": 5,
"decision_policy_version": "0.1.0"
},
"validation": {
"support": 72,
"positive_support": 27,
"precision": 0.5192307692307693,
"recall": 1.0,
"f1": 0.6835443037974683,
"macro_f1": 0.6494644595910419,
"average_precision": 0.7229152368252679,
"confusion_matrix": [
[
20,
25
],
[
0,
27
]
],
"coverage": 0.4305555555555556,
"review_rate": 0.5694444444444444,
"review_count": 41,
"redundant_count": 8,
"not_redundant_count": 23,
"selective_precision": 1.0,
"negative_predictive_value": 0.9130434782608695
}
},
{
"name": "contrastive",
"directory": "artifacts/extended-42/contrastive/model",
"kind": "cosine",
"policy": {
"binary_threshold": 0.5507670044898987,
"thresholds": {
"low": 0.5904079675674438,
"high": 0.8942374587059021,
"positive_enabled": true,
"negative_enabled": true
},
"selection_split": "validation",
"precision_target": 0.9,
"negative_predictive_value_target": 0.9,
"minimum_support": 5,
"decision_policy_version": "0.1.0"
},
"validation": {
"support": 72,
"positive_support": 27,
"precision": 0.54,
"recall": 1.0,
"f1": 0.7012987012987013,
"macro_f1": 0.6790075596045746,
"average_precision": 0.7632222551191008,
"confusion_matrix": [
[
22,
23
],
[
0,
27
]
],
"coverage": 0.5,
"review_rate": 0.5,
"review_count": 36,
"redundant_count": 10,
"not_redundant_count": 26,
"selective_precision": 0.9,
"negative_predictive_value": 0.9230769230769231
}
},
{
"name": "online_contrastive",
"directory": "artifacts/extended-42/online_contrastive/model",
"kind": "cosine",
"policy": {
"binary_threshold": 0.7588188052177429,
"thresholds": {
"low": 0.6037070751190186,
"high": 0.8967244625091553,
"positive_enabled": true,
"negative_enabled": true
},
"selection_split": "validation",
"precision_target": 0.9,
"negative_predictive_value_target": 0.9,
"minimum_support": 5,
"decision_policy_version": "0.1.0"
},
"validation": {
"support": 72,
"positive_support": 27,
"precision": 0.6774193548387096,
"recall": 0.7777777777777778,
"f1": 0.7241379310344828,
"macro_f1": 0.7690457097032879,
"average_precision": 0.7662009908295464,
"confusion_matrix": [
[
35,
10
],
[
6,
21
]
],
"coverage": 0.5,
"review_rate": 0.5,
"review_count": 36,
"redundant_count": 11,
"not_redundant_count": 25,
"selective_precision": 0.9090909090909091,
"negative_predictive_value": 0.92
}
},
{
"name": "cosent_balanced",
"directory": "artifacts/extended-42/cosent_balanced/model",
"kind": "cosine",
"policy": {
"binary_threshold": 0.6639197468757629,
"thresholds": {
"low": 0.4964098632335663,
"high": 0.8445258140563965,
"positive_enabled": true,
"negative_enabled": true
},
"selection_split": "validation",
"precision_target": 0.9,
"negative_predictive_value_target": 0.9,
"minimum_support": 5,
"decision_policy_version": "0.1.0"
},
"validation": {
"support": 72,
"positive_support": 27,
"precision": 0.5945945945945946,
"recall": 0.8148148148148148,
"f1": 0.6875,
"macro_f1": 0.71875,
"average_precision": 0.7527715859363884,
"confusion_matrix": [
[
30,
15
],
[
5,
22
]
],
"coverage": 0.4722222222222222,
"review_rate": 0.5277777777777778,
"review_count": 38,
"redundant_count": 10,
"not_redundant_count": 24,
"selective_precision": 0.9,
"negative_predictive_value": 0.9166666666666666
}
},
{
"name": "cosent_frozen",
"directory": "artifacts/extended-42/cosent_frozen/model",
"kind": "cosine",
"policy": {
"binary_threshold": 0.47448647022247314,
"thresholds": {
"low": 0.4975001811981201,
"high": 0.8474057912826538,
"positive_enabled": true,
"negative_enabled": true
},
"selection_split": "validation",
"precision_target": 0.9,
"negative_predictive_value_target": 0.9,
"minimum_support": 5,
"decision_policy_version": "0.1.0"
},
"validation": {
"support": 72,
"positive_support": 27,
"precision": 0.5294117647058824,
"recall": 1.0,
"f1": 0.6923076923076923,
"macro_f1": 0.6643356643356644,
"average_precision": 0.7605507017018385,
"confusion_matrix": [
[
21,
24
],
[
0,
27
]
],
"coverage": 0.5,
"review_rate": 0.5,
"review_count": 36,
"redundant_count": 11,
"not_redundant_count": 25,
"selective_precision": 0.9090909090909091,
"negative_predictive_value": 0.92
}
},
{
"name": "cross_encoder",
"directory": "artifacts/extended-42/cross_encoder/model",
"kind": "cross_encoder",
"policy": {
"binary_threshold": 0.40319615602493286,
"thresholds": {
"low": 0.20058020949363708,
"high": 1.0,
"positive_enabled": false,
"negative_enabled": true
},
"selection_split": "validation",
"precision_target": 0.9,
"negative_predictive_value_target": 0.9,
"minimum_support": 5,
"decision_policy_version": "0.1.0"
},
"validation": {
"support": 72,
"positive_support": 27,
"precision": 0.6875,
"recall": 0.8148148148148148,
"f1": 0.7457627118644068,
"macro_f1": 0.7846460618145563,
"average_precision": 0.7011072722166704,
"confusion_matrix": [
[
35,
10
],
[
5,
22
]
],
"coverage": 0.4444444444444444,
"review_rate": 0.5555555555555556,
"review_count": 40,
"redundant_count": 0,
"not_redundant_count": 32,
"selective_precision": null,
"negative_predictive_value": 0.90625
}
},
{
"name": "cross_encoder_frozen",
"directory": "artifacts/extended-42/cross_encoder_frozen/model",
"kind": "cross_encoder",
"policy": {
"binary_threshold": 0.43186578154563904,
"thresholds": {
"low": 0.15257886052131653,
"high": 1.0,
"positive_enabled": false,
"negative_enabled": true
},
"selection_split": "validation",
"precision_target": 0.9,
"negative_predictive_value_target": 0.9,
"minimum_support": 5,
"decision_policy_version": "0.1.0"
},
"validation": {
"support": 72,
"positive_support": 27,
"precision": 0.6571428571428571,
"recall": 0.8518518518518519,
"f1": 0.7419354838709677,
"macro_f1": 0.7734067663257278,
"average_precision": 0.6770074245823222,
"confusion_matrix": [
[
33,
12
],
[
4,
23
]
],
"coverage": 0.2916666666666667,
"review_rate": 0.7083333333333334,
"review_count": 51,
"redundant_count": 0,
"not_redundant_count": 21,
"selective_precision": null,
"negative_predictive_value": 0.9047619047619048
}
}
],
"skipped": [
{
"technique": "pair feature classifier",
"reason": "A tool write was blocked; feature baseline was not implemented or evaluated"
}
],
"config": {
"base_revision": "e8f8c211226b894fcb81acc59f3b34ba3efd5f42",
"base_directory": "artifacts/base",
"initial_directory": "artifacts/run-42/selected-model",
"dataset": "data/v01",
"refinement_dataset": "data/refinement",
"fresh_holdout": "data/v02",
"seed": 42,
"batch_size": 8,
"gradient_accumulation": 2,
"epochs": 3,
"learning_rate": 1e-05,
"loss": "cosent",
"selection": "validation average_precision, then binary F1; earliest tie",
"bi_candidates": [
{
"name": "contrastive",
"loss": "contrastive",
"margin": 0.5
},
{
"name": "online_contrastive",
"loss": "online_contrastive",
"margin": 0.5,
"batch_size": 16
},
{
"name": "cosent_balanced",
"loss": "cosent",
"learning_rate": 5e-06,
"sampling": "balanced"
},
{
"name": "cosent_frozen",
"loss": "cosent",
"epochs": 4,
"freeze_embeddings": true,
"freeze_lower_layers": 6
}
],
"cross_candidates": [
{
"name": "cross_encoder",
"epochs": 6,
"learning_rate": 2e-05
},
{
"name": "cross_encoder_frozen",
"epochs": 6,
"learning_rate": 3e-05,
"freeze_embeddings": true,
"freeze_lower_layers": 6
}
],
"pair_feature_logistic_C": [
1.0,
10.0
],
"test_policy": "Fresh holdout scored once after all v02 selection; previous tests are diagnostics only"
},
"fresh_holdout": {
"groups": 24,
"pairs": 48,
"sha256": {
"test.jsonl": "59af8f19c4c157a32b818ee021d34f02fa6178077baf577a37e3c89b603d9931"
},
"source_sha256": "e9f04b8b9385f2906eb56587e9a822ee344a4a162f94b9d2e9a7237dfb23fa95",
"provenance": {
"author": "same authoring AI",
"human_reviewed": false,
"license": "CC-BY-4.0",
"purpose": "Fresh scenarios frozen before v02 selection; old tests are diagnostic only",
"limitation": "General design informed by earlier aggregate errors; not independent human authorship"
}
},
"dataset": {
"schema_version": "0.1.0",
"seed": 42,
"groups": 60,
"pairs": 480,
"groups_by_split": {
"train": 42,
"test": 9,
"validation": 9
},
"source_sha256": "4539d298a41c756779270c7ab5f69a179249d9dc89e711fd1c530aa37549c6b1",
"sha256": {
"train.jsonl": "f1a038ea8e5acd0319d15100e8574aa95e1638190ad81adf58a89fa821150d8e",
"validation.jsonl": "12083d803d6c13a295c63eaa8bb311e7a0fa07e407374bb13af1567877166696",
"test.jsonl": "c857897c9a5b3a72d33927540810a625f5bc4c2ab8f7ea2abf55d54933d71bbd",
"all.jsonl": "32c4154fb356a73d28fadd52339a619f5efc288c024d5dadc437ee4d680ff882"
},
"provenance": {
"source": "scenarios.psv",
"author": "Codex authoring AI",
"reviewer": "same authoring AI; no independent or human review",
"human_reviewed": false,
"license": "CC-BY-4.0",
"limitation": "Shared authored constructions; not representative of real CVs"
}
},
"refinement_dataset": {
"schema_version": "0.1.0",
"seed": 42,
"groups": 20,
"pairs": 160,
"groups_by_split": {
"train": 20
},
"source_sha256": "cbf531c96059f20f5657de32435272623e8de46e1d08daadaeae206ebcdad2ad",
"sha256": {
"train.jsonl": "6b3cb804cf877c500aa567b0e583afe9dc684cd5c9fe567cb25593613e28831a"
},
"provenance": {
"source": "refinement.psv",
"author": "Codex authoring AI",
"reviewer": "same authoring AI; no independent or human review",
"human_reviewed": false,
"license": "CC-BY-4.0",
"limitation": "Shared authored constructions; not representative of real CVs"
}
},
"test_evaluated": false,
"selection_rule": "validation average_precision, then binary F1; earliest tie"
}