File size: 7,048 Bytes
f0b6377
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
{
  "model_version": "0.2.0-experimental",
  "model_revision": "sha256:6c29d90e982da340a9dfc970df69d4455b7669bef929d2a9b84d0cc378effdf1",
  "decision_policy_version": "0.1.0",
  "base_revision": "e8f8c211226b894fcb81acc59f3b34ba3efd5f42",
  "training": {
    "config": {
      "base_revision": "e8f8c211226b894fcb81acc59f3b34ba3efd5f42",
      "base_directory": "artifacts/base",
      "initial_directory": "artifacts/run-42/selected-model",
      "dataset": "data/v01",
      "refinement_dataset": "data/refinement",
      "fresh_holdout": "data/v02",
      "seed": 42,
      "batch_size": 16,
      "gradient_accumulation": 2,
      "epochs": 3,
      "learning_rate": 1e-05,
      "loss": "online_contrastive",
      "selection": "validation average_precision, then binary F1; earliest tie",
      "bi_candidates": [
        {
          "name": "contrastive",
          "loss": "contrastive",
          "margin": 0.5
        },
        {
          "name": "online_contrastive",
          "loss": "online_contrastive",
          "margin": 0.5,
          "batch_size": 16
        },
        {
          "name": "cosent_balanced",
          "loss": "cosent",
          "learning_rate": 5e-06,
          "sampling": "balanced"
        },
        {
          "name": "cosent_frozen",
          "loss": "cosent",
          "epochs": 4,
          "freeze_embeddings": true,
          "freeze_lower_layers": 6
        }
      ],
      "cross_candidates": [
        {
          "name": "cross_encoder",
          "epochs": 6,
          "learning_rate": 2e-05
        },
        {
          "name": "cross_encoder_frozen",
          "epochs": 6,
          "learning_rate": 3e-05,
          "freeze_embeddings": true,
          "freeze_lower_layers": 6
        }
      ],
      "pair_feature_logistic_C": [
        1.0,
        10.0
      ],
      "test_policy": "Fresh holdout scored once after all v02 selection; previous tests are diagnostics only",
      "name": "online_contrastive",
      "margin": 0.5,
      "model_version": "0.2.0-online_contrastive"
    },
    "training_pairs": 496,
    "validation_pairs": 72,
    "selected_epoch": 2,
    "selected_parameter_delta_l2": 0.023539088666439056,
    "optimizer_steps": 48,
    "history": [
      {
        "epoch": 1,
        "train_loss": 0.17605107638143724,
        "validation": {
          "support": 72,
          "positive_support": 27,
          "precision": 0.6363636363636364,
          "recall": 0.7777777777777778,
          "f1": 0.7,
          "macro_f1": 0.7428571428571429,
          "average_precision": 0.7524502288485848,
          "confusion_matrix": [
            [
              33,
              12
            ],
            [
              6,
              21
            ]
          ],
          "coverage": 0.5,
          "review_rate": 0.5,
          "review_count": 36,
          "redundant_count": 11,
          "not_redundant_count": 25,
          "selective_precision": 0.9090909090909091,
          "negative_predictive_value": 0.92
        }
      },
      {
        "epoch": 2,
        "train_loss": 0.13096907350324816,
        "validation": {
          "support": 72,
          "positive_support": 27,
          "precision": 0.6774193548387096,
          "recall": 0.7777777777777778,
          "f1": 0.7241379310344828,
          "macro_f1": 0.7690457097032879,
          "average_precision": 0.7662009908295464,
          "confusion_matrix": [
            [
              35,
              10
            ],
            [
              6,
              21
            ]
          ],
          "coverage": 0.5,
          "review_rate": 0.5,
          "review_count": 36,
          "redundant_count": 11,
          "not_redundant_count": 25,
          "selective_precision": 0.9090909090909091,
          "negative_predictive_value": 0.92
        }
      },
      {
        "epoch": 3,
        "train_loss": 0.12419637556998961,
        "validation": {
          "support": 72,
          "positive_support": 27,
          "precision": 0.7,
          "recall": 0.7777777777777778,
          "f1": 0.7368421052631579,
          "macro_f1": 0.7822141560798548,
          "average_precision": 0.7639659181489029,
          "confusion_matrix": [
            [
              36,
              9
            ],
            [
              6,
              21
            ]
          ],
          "coverage": 0.5,
          "review_rate": 0.5,
          "review_count": 36,
          "redundant_count": 11,
          "not_redundant_count": 25,
          "selective_precision": 0.9090909090909091,
          "negative_predictive_value": 0.92
        }
      }
    ],
    "policy": {
      "binary_threshold": 0.7588188052177429,
      "thresholds": {
        "low": 0.6037070751190186,
        "high": 0.8967244625091553,
        "positive_enabled": true,
        "negative_enabled": true
      },
      "selection_split": "validation",
      "precision_target": 0.9,
      "negative_predictive_value_target": 0.9,
      "minimum_support": 5,
      "decision_policy_version": "0.1.0"
    },
    "selection": "validation average_precision, then binary F1; earliest tie",
    "duration_seconds": 100.0618496590032,
    "environment": {
      "python": "3.11.14",
      "platform": "Linux-7.1.8+deb13-amd64-x86_64-with-glibc2.41",
      "code_commit": "28071a1a31a0090f70c616f8a1680cd1da3e97f1",
      "source_sha256": {
        "cli.py": "8462dd1c90e06165826a5053c740c2e4379bc4db1a3fafacb38eccc866f007ab",
        "decision.py": "ce69d18701fb687f0fd94217a121f7ebdd29e9397c0c7a18c899dcede96f5e2a",
        "encoder.py": "027fa1f8ed388662e13fb3c8390e2024f20ef10f6e96923b46bf4074ffd49905",
        "data.py": "5b8143e617f9166f13159f96081b9048b24fe6d80289b181fc165f1c307b7dde",
        "experiment.py": "d6cac58f617129f20a3518724d53678d6ef204fee35a9e998d1dc1aafd8af891",
        "training.py": "2a375b311b3a9e3e5d773e0b0f974ff859dd4f232d639c8176302128142aa8c0",
        "__main__.py": "b466d3224c1cfa96b82f891d496e36e7bd4ad1eb70fcc530b45c29ce62d3f9c4",
        "pair_features.py": "b995a5c8d6314a4367949b232ba7060829c16ce41d9de3926dab50ee5619bb98",
        "cross_encoder.py": "dded92e08e97b34e33c21d2eba017b7bfe023e19eed89a3a8b836d4ea61697f8",
        "schemas.py": "ecb86544d6f71b5dad802e345438763050ac831732a6c0a126c80c921cfa7aef",
        "evaluation.py": "4880d0ee9fcf6fc15163204b2edcf4856dc06dfe5aaa3930a9dca236b16d0fd0",
        "__init__.py": "6f77bebe829dfcc6c5545b6ed57e87ea355244777e5fe8b320589566fd5f9791",
        "dataset.py": "40c6153527279d43d66d60de10dc0cb1bfbcba07df11a64d18ad810d56bfb149"
      },
      "deterministic_warn_only": true,
      "cublas_workspace_config": ":4096:8",
      "gpu": "NVIDIA RTX A1000 6GB Laptop GPU",
      "packages": {
        "torch": "2.6.0",
        "sentence-transformers": "5.1.2",
        "transformers": "4.57.1",
        "numpy": "2.3.4",
        "scikit-learn": "1.7.2"
      }
    }
  },
  "decision_source": "cosine",
  "selected_embedding": "online_contrastive",
  "selected_decision_model": "online_contrastive"
}