SS3M commited on
Commit
70a11a0
·
verified ·
1 Parent(s): 6cfa166

Upload 11_12_13_lr_17's state dict

Browse files
.gitattributes CHANGED
@@ -79,3 +79,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
79
  10_lr_stronger_proj_residual_13/logs/10_lr_stronger_proj_residual_13_log_plot.jpg filter=lfs diff=lfs merge=lfs -text
80
  11_12_lr_14/logs/11_12_lr_14_log_plot.jpg filter=lfs diff=lfs merge=lfs -text
81
  11_13_lr_15/logs/11_13_lr_15_log_plot.jpg filter=lfs diff=lfs merge=lfs -text
 
 
79
  10_lr_stronger_proj_residual_13/logs/10_lr_stronger_proj_residual_13_log_plot.jpg filter=lfs diff=lfs merge=lfs -text
80
  11_12_lr_14/logs/11_12_lr_14_log_plot.jpg filter=lfs diff=lfs merge=lfs -text
81
  11_13_lr_15/logs/11_13_lr_15_log_plot.jpg filter=lfs diff=lfs merge=lfs -text
82
+ 11_12_13_lr_17/logs/11_12_13_lr_17_log_plot.jpg filter=lfs diff=lfs merge=lfs -text
11_12_13_lr_17/11_12_13_lr_17.py ADDED
@@ -0,0 +1,1596 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # %% [code]
2
+ get_ipython().system('pip install evaluate seqeval underthesea positional-encodings[pytorch]')
3
+
4
+ # %% [code]
5
+ import warnings
6
+ warnings.filterwarnings('ignore')
7
+
8
+ import torch
9
+ import torch.nn as nn
10
+ import torch.optim as optim
11
+ from torch.utils.data import Dataset, TensorDataset, DataLoader
12
+ import torch.nn.functional as F
13
+ import albumentations as albu
14
+ from transformers import AutoTokenizer, AutoModel
15
+ import torch.distributed as dist
16
+ from torch.nn.parallel import DistributedDataParallel as DDP
17
+ from positional_encodings.torch_encodings import PositionalEncoding1D
18
+
19
+ from sklearn.metrics import f1_score
20
+ from sklearn.preprocessing import MinMaxScaler, StandardScaler
21
+ from scipy.spatial.transform import Rotation as R
22
+ from sklearn.model_selection import KFold, StratifiedGroupKFold, GroupKFold, StratifiedKFold
23
+ from sklearn.metrics import precision_recall_fscore_support
24
+ from timm.utils import ModelEmaV3
25
+ import timm
26
+
27
+ import os
28
+ import gc
29
+ import json
30
+ from pathlib import Path
31
+ import pickle
32
+ from tqdm.auto import tqdm
33
+ import copy
34
+ import numpy as np
35
+ import pandas as pd
36
+ import polars as pl
37
+ from PIL import Image
38
+ import time
39
+ from tqdm import tqdm
40
+ from matplotlib import pyplot as plt
41
+ import seaborn as sns
42
+ from multiprocessing import Manager as MemoryManager
43
+ from functools import lru_cache
44
+ import shutil
45
+ import glob
46
+ import cv2
47
+ import random
48
+ import re
49
+ import joblib
50
+ import math
51
+ from huggingface_hub import HfApi, snapshot_download
52
+ import evaluate
53
+ from underthesea import word_tokenize as vi_tokenize_tool
54
+ import spacy
55
+ en_tokenize_tool = spacy.load("en_core_web_sm")
56
+ from collections import defaultdict, Counter
57
+
58
+ # %% [code]
59
+ file_path = "/kaggle/input/notebooks/sambui22022517/kltn-lr-bm25/bm25_scores.npy"
60
+
61
+ bm25_scores = np.load(file_path, allow_pickle=True)
62
+ print(bm25_scores.shape)
63
+
64
+ # %% [code]
65
+ # Global config
66
+ SEEDS = [26092004]
67
+ topk = 1
68
+ nfolds = 5
69
+ only_fold_idx = 0
70
+ test_only = 0
71
+ debug_only = 0
72
+
73
+ # Config thư mục
74
+ dataset = 'kltn/raw' # vhe, bkee, casie, kltn/only_issues, kltn/only_actions, kltn/raw
75
+ root_dir = f'/kaggle/input/notebooks/sambui22022517/kltn-data/{dataset}' ## Thư mục chứa file train, val, test
76
+ train_dir = f'{root_dir}'
77
+ # val_dir = f'{root_dir}/val'
78
+ test_dir = f'{root_dir}'
79
+
80
+ # Config checkpoints
81
+
82
+ # Config training
83
+ epochs = 18 if not debug_only else 2
84
+ batch_size = 32
85
+ device = "cuda" if torch.cuda.is_available() else "cpu"
86
+ # # Thêm biến toàn cục nào đó vào đây
87
+ repo_name = 'SS3M/kltn-experiments'
88
+ state_dict_save_name = "11_12_13_lr_17"
89
+ checkpoints_dir = state_dict_save_name
90
+ pretrained_dir = "/kaggle/working"
91
+ os.makedirs(f'{checkpoints_dir}', exist_ok=True)
92
+
93
+ backbone_model_name = "bert-base-uncased" if dataset == "casie" else "vinai/phobert-base"
94
+ word_tokenize = lambda text: [token.text for token in en_tokenize_tool(text)] if dataset == "casie" else vi_tokenize_tool(text)
95
+ max_len_dict = {
96
+ 'kltn/raw': 256,
97
+ 'kltn/only_issues': 52,
98
+ 'kltn/only_actions': 69,
99
+ 'vhe': 51,
100
+ 'bkee': 62,
101
+ 'casie': 40,
102
+ }
103
+ zero_events_rate_dict = {
104
+ 'kltn/raw': 1000,
105
+ 'kltn/only_issues': 0,
106
+ 'kltn/only_actions': 0.2,
107
+ 'vhe': 1000, # mean keep all zero-events samples
108
+ 'bkee': 1000,
109
+ 'casie': 1000,
110
+ }
111
+
112
+ max_len = max_len_dict[dataset]
113
+ max_n_parts = 2
114
+ max_span_len = 14
115
+ n_negs = 5 * 20
116
+ zero_events_rate = zero_events_rate_dict[dataset]
117
+
118
+ # Trainer
119
+ trainer_params = {
120
+ "training_time": "00:11:30:00",
121
+ "eval_mode": "max",
122
+ "topk": topk,
123
+ "save_name": state_dict_save_name,
124
+ "save_best": True,
125
+ "save_last": True,
126
+ "device": device,
127
+ "logging": True,
128
+ "logging_file": True,
129
+ "checkpoints_dir": checkpoints_dir,
130
+ "early_stopping": 30,
131
+ "eval_from_ratio": 0.4,
132
+ "eval_every": 1,
133
+ "schedule_in_step": False,
134
+ "use_ema": True,
135
+ "ema_from_ratio": 0.3,
136
+ "ema_decay": 0.9995,
137
+ "max_grad_norm": 200.0,
138
+ "return_best": True,
139
+ "return_last": True,
140
+ }
141
+
142
+ # Memory
143
+ train_memory_params = {
144
+ 'max_len': max_len,
145
+ 'max_n_parts': max_n_parts,
146
+ 'n_negs': n_negs,
147
+ }
148
+ val_memory_params = {
149
+ 'max_len': max_len,
150
+ 'max_n_parts': max_n_parts,
151
+ 'n_negs': n_negs,
152
+ }
153
+ corpus_memory_params = {
154
+ 'max_len': max_len,
155
+ 'max_n_parts': max_n_parts,
156
+ }
157
+
158
+ # Data Loader
159
+ def seed_worker(worker_id):
160
+ worker_seed = torch.initial_seed() % 2**32
161
+ np.random.seed(worker_seed)
162
+ random.seed(worker_seed)
163
+
164
+ train_loader_params = {
165
+ 'batch_size': batch_size,
166
+ 'shuffle': True,
167
+ 'pin_memory':True,
168
+ 'num_workers': 2,
169
+ 'drop_last': False,
170
+ 'worker_init_fn': seed_worker,
171
+ 'persistent_workers': False,
172
+ }
173
+ val_loader_params = {
174
+ 'batch_size': batch_size,
175
+ 'shuffle': False,
176
+ 'pin_memory':True,
177
+ 'num_workers': 1,
178
+ 'drop_last': False,
179
+ 'worker_init_fn': seed_worker,
180
+ 'persistent_workers': False,
181
+ }
182
+
183
+ # Model
184
+ model_params = {
185
+ 'backbone_name': backbone_model_name,
186
+ 'projection_dim': 256,
187
+ 'normalize': True,
188
+ }
189
+
190
+ # Loss Func
191
+ loss_func_params = {
192
+ 'lambda_contrastive': 1.0,
193
+ 'lambda_triplet': 0.5,
194
+ }
195
+ eval_func_params = {}
196
+
197
+ # Optim
198
+ optim_params = {
199
+ 'name': 'AdamW',
200
+ 'lr': 1e-4,
201
+ 'weight_decay': 1e-4,
202
+ }
203
+ scheduler_params = {
204
+ 'name': 'CosineAnnealingLR',
205
+ 'T_max': 20, # Số epoch để hoàn thành một chu kỳ giảm LR
206
+ 'eta_min': 1e-6 # Learning rate nhỏ nhất trong chu kỳ
207
+ }
208
+
209
+ # %% [code]
210
+ def set_seed(seed=42):
211
+ random.seed(seed)
212
+ np.random.seed(seed)
213
+ torch.manual_seed(seed)
214
+ torch.cuda.manual_seed(seed)
215
+ torch.cuda.manual_seed_all(seed) # if using multi-GPU
216
+ torch.use_deterministic_algorithms(False)
217
+ torch.backends.cudnn.deterministic = True
218
+ torch.backends.cudnn.benchmark = False
219
+ os.environ['PYTHONHASHSEED'] = str(seed)
220
+
221
+ # %% [code]
222
+ class CustomLoss(nn.Module):
223
+ def __init__(
224
+ self,
225
+ temperature=0.05,
226
+ margin=0.2,
227
+ lambda_contrastive=1.0,
228
+ lambda_triplet=0.5,
229
+ ):
230
+ super().__init__()
231
+
232
+ self.temperature = temperature
233
+ self.margin = margin
234
+
235
+ self.lambda_contrastive = lambda_contrastive
236
+ self.lambda_triplet = lambda_triplet
237
+
238
+ def forward(
239
+ self,
240
+ encoded_text,
241
+ encoded_pos,
242
+ encoded_neg,
243
+ pos_mask
244
+ ):
245
+ loss_contrastive = self.multi_pos_contrastive_loss(encoded_text, encoded_pos, encoded_neg, pos_mask)
246
+ loss_triplet = self.hardest_triplet_loss(encoded_text, encoded_pos, encoded_neg, pos_mask)
247
+
248
+ total_loss = (
249
+ self.lambda_contrastive * loss_contrastive +
250
+ self.lambda_triplet * loss_triplet
251
+ )
252
+
253
+ return {
254
+ "total": total_loss,
255
+ "contrastive_loss": loss_contrastive,
256
+ "triplet_loss": loss_triplet,
257
+ }
258
+
259
+ def multi_pos_contrastive_loss(self, q, pos, neg, pos_mask):
260
+ B, P, D = pos.shape
261
+ N = neg.shape[1]
262
+
263
+ # ===== concat docs =====
264
+ docs = torch.cat([pos, neg], dim=1) # [B, P+N, D]
265
+
266
+ # ===== similarity =====
267
+ logits = torch.matmul(q.unsqueeze(1), docs.transpose(1, 2)).squeeze(1)
268
+ logits = logits / self.temperature # [B, P+N]
269
+
270
+ # ===== labels =====
271
+ labels = torch.zeros_like(logits)
272
+ labels[:, :P] = pos_mask # chỉ pos hợp lệ
273
+
274
+ # ===== log-softmax =====
275
+ log_prob = logits - torch.logsumexp(logits, dim=1, keepdim=True)
276
+
277
+ # ===== normalize theo số pos thật =====
278
+ pos_count = pos_mask.sum(dim=1).clamp(min=1)
279
+
280
+ loss = -(labels * log_prob).sum(dim=1) / pos_count
281
+
282
+ return loss.mean()
283
+
284
+ def hardest_triplet_loss(self, q, pos, neg, pos_mask):
285
+ # ===== similarity =====
286
+ pos_sim = torch.matmul(q.unsqueeze(1), pos.transpose(1, 2)).squeeze(1) # [B, P]
287
+ neg_sim = torch.matmul(q.unsqueeze(1), neg.transpose(1, 2)).squeeze(1) # [B, N]
288
+
289
+ # ===== mask pos =====
290
+ pos_sim_masked = pos_sim.clone()
291
+ pos_sim_masked[pos_mask == 0] = float('inf') # loại pad
292
+
293
+ # ===== hardest =====
294
+ hardest_pos = pos_sim_masked.min(dim=1).values
295
+ hardest_neg = neg_sim.max(dim=1).values
296
+
297
+ # ===== loss =====
298
+ loss = F.relu(self.margin + hardest_neg - hardest_pos)
299
+
300
+ return loss.mean()
301
+
302
+ # %% [code]
303
+ class CustomEvalFn(nn.Module):
304
+ def __init__(self):
305
+ super().__init__()
306
+
307
+ def forward(self, pred_topk, real_topk):
308
+ """
309
+ pred_topk: List[List[int]] shape [B, K]
310
+ real_topk: List[List[int]] shape [B, Ki]
311
+ """
312
+
313
+ B = len(pred_topk)
314
+
315
+ total_recall = 0.0
316
+ total_map = 0.0
317
+ total_mrp = 0.0
318
+
319
+ for i in range(B):
320
+ preds = pred_topk[i]
321
+ gts = set(real_topk[i])
322
+
323
+ # ===== Recall@K =====
324
+ hit = any(p in gts for p in preds)
325
+ total_recall += 1.0 if hit else 0.0
326
+
327
+ # ===== AP =====
328
+ num_hits = 0
329
+ ap = 0.0
330
+
331
+ for rank, p in enumerate(preds, start=1):
332
+ if p in gts:
333
+ num_hits += 1
334
+ precision_at_rank = num_hits / rank
335
+ ap += precision_at_rank
336
+
337
+ if len(gts) > 0:
338
+ ap /= len(gts)
339
+
340
+ total_map += ap
341
+
342
+ # ===== R-Precision =====
343
+ r = len(gts)
344
+
345
+ if r > 0:
346
+ top_r = preds[:r]
347
+
348
+ tp_r = sum(p in gts for p in top_r)
349
+
350
+ rp = tp_r / r
351
+ else:
352
+ rp = 0.0
353
+
354
+ total_mrp += rp
355
+
356
+ recall = total_recall / B
357
+ mAP = total_map / B
358
+ mRP = total_mrp / B
359
+
360
+ return {
361
+ "recall": recall,
362
+ "mAP": mAP,
363
+ "mRP": mRP,
364
+ }
365
+
366
+ # %% [code]
367
+ class EncodeModel(nn.Module):
368
+ def __init__(
369
+ self,
370
+ backbone_name,
371
+ projection_dim,
372
+ normalize=True,
373
+ dropout=0.1
374
+ ):
375
+ super().__init__()
376
+
377
+ self.encoder = AutoModel.from_pretrained(backbone_name)
378
+
379
+ hidden_size = self.encoder.config.hidden_size
380
+
381
+ # mạnh hơn single linear
382
+ self.proj = nn.Sequential(
383
+ nn.Linear(hidden_size, hidden_size),
384
+ nn.GELU(),
385
+ nn.Dropout(dropout),
386
+ nn.Linear(hidden_size, projection_dim)
387
+ )
388
+
389
+ # residual projection
390
+ self.residual_proj = (
391
+ nn.Identity()
392
+ if hidden_size == projection_dim
393
+ else nn.Linear(hidden_size, projection_dim)
394
+ )
395
+ self.chunk_attn = nn.Linear(hidden_size, 1)
396
+
397
+ self.normalize = normalize
398
+
399
+ def mean_pooling(self, hidden, attention_mask):
400
+ mask = attention_mask.unsqueeze(-1).float() # [N, L, 1]
401
+ pooled = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-6)
402
+
403
+ return pooled
404
+
405
+ def chunk_attention_pooling(self, chunk_repr, dim):
406
+ scores = self.chunk_attn(chunk_repr)
407
+
408
+ weights = torch.softmax(scores, dim=dim)
409
+
410
+ pooled = (chunk_repr * weights).sum(dim=dim)
411
+
412
+ return pooled
413
+
414
+ def forward(self, input_ids, attention_mask, is_query=True):
415
+
416
+ if is_query:
417
+ # [B, n_parts, L]
418
+ B, n_parts, L = input_ids.shape
419
+
420
+ input_ids = input_ids.view(-1, L)
421
+ attention_mask = attention_mask.view(-1, L)
422
+
423
+ outputs = self.encoder(
424
+ input_ids=input_ids,
425
+ attention_mask=attention_mask
426
+ )
427
+
428
+ hidden = outputs.last_hidden_state # [B*n_parts, L, H]
429
+
430
+ chunk_repr = self.mean_pooling(hidden, attention_mask) # [B*n_parts, H]
431
+ chunk_repr = chunk_repr.view(B, n_parts, -1) # [B, n_parts, H]
432
+ final_repr = self.chunk_attention_pooling(chunk_repr, dim=1) # [B, H]
433
+ # [B, H]
434
+
435
+ else:
436
+ # [B, K, n_parts, L]
437
+ B, K, n_parts, L = input_ids.shape
438
+
439
+ input_ids = input_ids.view(-1, L)
440
+ attention_mask = attention_mask.view(-1, L)
441
+
442
+ outputs = self.encoder(
443
+ input_ids=input_ids,
444
+ attention_mask=attention_mask
445
+ )
446
+
447
+ hidden = outputs.last_hidden_state
448
+
449
+ chunk_repr = self.mean_pooling(hidden, attention_mask) # [B*K*n_parts, H]
450
+ chunk_repr = chunk_repr.view(B, K, n_parts, -1) # [B, K, n_parts, H]
451
+ final_repr = self.chunk_attention_pooling(chunk_repr, dim=2) # [B, K, H]
452
+
453
+ # residual MLP projection
454
+ emb = self.proj(final_repr) + self.residual_proj(final_repr)
455
+
456
+ if self.normalize:
457
+ emb = F.normalize(emb, dim=-1)
458
+
459
+ return emb
460
+
461
+ def test_model():
462
+ model = nn.DataParallel(EncodeModel('vinai/phobert-base', 256, True)).to(device)
463
+ model.eval()
464
+
465
+ bz = 32
466
+ vocab_size = 1000
467
+ qi = torch.randint(0, vocab_size, (bz, 1, 256)).to(device)
468
+ qa = torch.ones(bz, 1, 256).to(device)
469
+ di = torch.randint(0, vocab_size, (bz, 5, 2, 256)).to(device)
470
+ da = torch.ones(bz, 5, 2, 256).to(device)
471
+
472
+ st = time.time()
473
+ with torch.no_grad():
474
+ encoded_text = model(qi, qa, is_query=True)
475
+ encoded_pos = model(di, da, is_query=False)
476
+ encoded_neg = model(di, da, is_query=False)
477
+ print(encoded_text.shape, encoded_pos.shape, encoded_neg.shape)
478
+ print(time.time() - st)
479
+
480
+ del model, qi, qa, di, da, encoded_text, encoded_pos, encoded_neg
481
+ torch.cuda.empty_cache()
482
+ gc.collect()
483
+ test_model()
484
+
485
+ # %% [code]
486
+ def configure_optimizers(network, optim_params, scheduler_params):
487
+ try:
488
+ optim_params = copy.copy(optim_params)
489
+ scheduler_params = copy.copy(scheduler_params)
490
+
491
+ optim_name = optim_params.pop('name')
492
+ scheduler_name = scheduler_params.pop('name')
493
+
494
+ optimizer_cls = globals().get(optim_name) or getattr(optim, optim_name, None)
495
+ scheduler_cls = globals().get(scheduler_name) or getattr(optim.lr_scheduler, scheduler_name, None)
496
+
497
+ if optimizer_cls is None:
498
+ raise ValueError(f"Optimizer '{optim_name}' is not available!")
499
+
500
+ optimizer = optimizer_cls(network.parameters(), **optim_params)
501
+
502
+ scheduler = None
503
+ if scheduler_params and scheduler_cls: # Chỉ tạo scheduler nếu có tham số
504
+ scheduler = scheduler_cls(optimizer, **scheduler_params)
505
+
506
+ return optimizer, scheduler
507
+
508
+ except KeyError as e:
509
+ raise ValueError(f"Missing {e} in config!!")
510
+
511
+ def freeze(self, model):
512
+ model.eval()
513
+ for param in model.parameters():
514
+ param.requires_grad = False
515
+
516
+ def unfreeze(self, model):
517
+ model.train()
518
+ for param in model.parameters():
519
+ param.requires_grad = True
520
+
521
+ def reduce_batch_size(loader, ratio=0.5):
522
+ new_bs = max(1, int(loader.batch_size * ratio))
523
+
524
+ shuffle = isinstance(loader.sampler, RandomSampler)
525
+
526
+ new_loader = DataLoader(
527
+ dataset=loader.dataset,
528
+ batch_size=new_bs,
529
+ shuffle=shuffle,
530
+ sampler=None if shuffle else loader.sampler,
531
+ num_workers=loader.num_workers,
532
+ collate_fn=loader.collate_fn,
533
+ pin_memory=loader.pin_memory,
534
+ drop_last=loader.drop_last,
535
+ timeout=loader.timeout,
536
+ worker_init_fn=loader.worker_init_fn,
537
+ multiprocessing_context=loader.multiprocessing_context,
538
+ generator=loader.generator,
539
+ prefetch_factor=loader.prefetch_factor if loader.num_workers > 0 else None,
540
+ persistent_workers=loader.persistent_workers,
541
+ pin_memory_device=loader.pin_memory_device
542
+ )
543
+
544
+ return new_loader
545
+
546
+ def list_to_tuple(x):
547
+ if isinstance(x, (list, tuple)):
548
+ return tuple(list_to_tuple(i) for i in x)
549
+ return x
550
+
551
+ def fmt(x):
552
+ if isinstance(x, float):
553
+ return round(x, 5)
554
+ if isinstance(x, dict):
555
+ return {k: fmt(v) for k, v in x.items()}
556
+ if isinstance(x, list):
557
+ return [fmt(v) for v in x]
558
+ return x
559
+
560
+ class ModelEmaV3Proxy(ModelEmaV3):
561
+ def __getattr__(self, name):
562
+ try:
563
+ return super().__getattr__(name)
564
+ except AttributeError:
565
+ return getattr(self.module, name)
566
+
567
+ class DataParallelProxy(nn.DataParallel):
568
+ def __getattr__(self, name):
569
+ try:
570
+ return super().__getattr__(name)
571
+ except AttributeError:
572
+ attr = getattr(self.module, name)
573
+
574
+ if callable(attr):
575
+ def wrapper(*args, **kwargs):
576
+ return self._parallel_apply_method(name, *args, **kwargs)
577
+ return wrapper
578
+
579
+ return attr
580
+
581
+ def _parallel_apply_method(self, method_name, *inputs, **kwargs):
582
+ if not self.device_ids:
583
+ return getattr(self.module, method_name)(*inputs, **kwargs)
584
+
585
+ inputs_scattered, kwargs_scattered = self.scatter(inputs, kwargs, self.device_ids)
586
+
587
+ replicas = self.replicate(self.module, self.device_ids)
588
+
589
+ outputs = self.parallel_apply(
590
+ [getattr(replica, method_name) for replica in replicas],
591
+ inputs_scattered,
592
+ kwargs_scattered
593
+ )
594
+
595
+ return self.gather(outputs, self.output_device)
596
+
597
+ class Trainer:
598
+ def __init__(
599
+ self, training_time="00:11:30:00", eval_mode="max", topk=1, save_name="network", save_best=True, save_last=False, max_grad_norm=200.0,
600
+ logging=0, logging_file=False, checkpoints_dir="", early_stopping=False, eval_from_ratio=-1, eval_every=1, device='cpu',
601
+ schedule_in_step=True, use_ema=True, ema_from_ratio=-1, ema_decay=0.999, return_best=True, return_last=True
602
+ ):
603
+ self.ema_net = None
604
+
605
+ self.training_time = self._time_str_to_seconds(training_time)
606
+ self.mode = eval_mode
607
+ self.topk = topk
608
+ self.device = device
609
+ self.logging = logging if logging < epochs else 1
610
+ self.logging_file = logging_file
611
+ self.checkpoints_dir = checkpoints_dir
612
+ self.early_stopping = early_stopping
613
+ self.eval_from_ratio = eval_from_ratio
614
+ self.eval_every = eval_every
615
+ self.save_name = save_name
616
+ self.save_best = save_best
617
+ self.save_last = save_last
618
+ self.return_best = return_best
619
+ self.return_last = return_last
620
+ self.max_grad_norm = max_grad_norm
621
+ self.schedule_in_step = schedule_in_step
622
+ self.use_ema = use_ema
623
+ self.ema_from_ratio = ema_from_ratio
624
+ self.ema_decay = ema_decay
625
+
626
+ self.best_stage = [[float('-inf') if self.mode == 'max' else float('inf'), None, None]]
627
+ self.grad_scaler = torch.amp.GradScaler(self.device, init_scale=1024.0)
628
+
629
+ def fit(self, network, optimizer, scheduler, loss_fn, epochs, train_loader, val_loader=None, corpus_loader=None, eval_fn=None, start_epoch=1, start_training_time=None, refresh_every=3):
630
+ if eval_fn is None:
631
+ if self.mode == "max":
632
+ eval_fn = lambda *x: -loss_fn(*x)
633
+ else:
634
+ eval_fn = lambda *x: loss_fn(*x)
635
+
636
+ if torch.cuda.device_count() > 1:
637
+ network = DataParallelProxy(network)
638
+ network = network.to(self.device)
639
+
640
+ if not start_training_time:
641
+ start_training_time = time.time()
642
+
643
+ start_ema = int(epochs * self.ema_from_ratio)
644
+ start_eval = int(epochs * self.eval_from_ratio)
645
+
646
+ if val_loader is None:
647
+ print(f'[Trainer CallBack] 📢 Không có Val Set, không thể đánh giá và Early Stopping!')
648
+ else:
649
+ model_to_use_str = 'mô hình EMA' if self.use_ema else 'mô hình gốc'
650
+ start_model_update_str = f'Bắt đầu cập nhật EMA từ epoch {start_epoch + start_ema}!' if self.use_ema else ''
651
+ print(f'[Trainer CallBack] 📢 Đánh giá bằng {model_to_use_str} từ epoch {start_epoch + start_eval}!', start_model_update_str)
652
+
653
+ training_log = {}
654
+ for epoch in range(start_epoch, epochs+start_epoch):
655
+ if self.use_ema and self.ema_net is None and epoch - start_epoch >= start_ema:
656
+ self.ema_net = ModelEmaV3Proxy(network, self.ema_decay, device=self.device)
657
+
658
+ try:
659
+ eval_net = self.ema_net if (self.use_ema and self.ema_net is not None) else network
660
+ if (epoch - start_epoch) % refresh_every == 0:
661
+ encoded_docs = self._get_encoded_docs(eval_net, corpus_loader)
662
+ print(f"[Trainer CallBack] 📢 Epoch {epoch}/{epochs}: Refresh Encoded Doc (refresh_every={refresh_every})!")
663
+ elif (epoch - start_epoch - start_eval) % self.eval_every == 0 and epoch - start_epoch >= start_eval:
664
+ encoded_docs = self._get_encoded_docs(eval_net, corpus_loader)
665
+ print(f"[Trainer CallBack] 📢 Epoch {epoch}/{epochs}: Refresh Encoded Doc (eval_every={self.eval_every})!")
666
+
667
+ train_loss_epoch, train_loss_epoch_dict = self._train_epoch(network, train_loader, optimizer, scheduler, loss_fn, encoded_docs)
668
+ logging_dict = {'lr': [group['lr'] for group in optimizer.param_groups], 'train_loss': train_loss_epoch}
669
+ logging_dict.update(train_loss_epoch_dict)
670
+
671
+ if val_loader is not None and epoch - start_epoch >= start_eval and (epoch - start_epoch - start_eval) % self.eval_every == 0:
672
+ val_score, val_score_dict, _ = self._eval_epoch(eval_net, val_loader, eval_fn, encoded_docs)
673
+ update = self._update_best_network(eval_net, val_score, epoch)
674
+ logging_dict.update({'val_score': val_score, 'best_score': self.best_stage[0][0], 'new_best_model': update})
675
+ logging_dict.update(val_score_dict)
676
+ if not self.schedule_in_step and scheduler:
677
+ scheduler.step()
678
+
679
+ except RuntimeError as e:
680
+ if "out of memory" in str(e).lower():
681
+ print(f"[Trainer CallBack] ⚠️ Epoch {epoch}/{epochs}: CUDA Out of Memory! Clearing GPU cache...")
682
+ torch.cuda.empty_cache()
683
+ gc.collect()
684
+ if torch.cuda.is_available():
685
+ torch.cuda.synchronize()
686
+ print(f"[Trainer CallBack] ✅ Epoch {epoch}/{epochs}: GPU memory cleared")
687
+
688
+ train_loader = reduce_batch_size(train_loader, ratio=0.5)
689
+ if val_loader is not None:
690
+ val_loader = reduce_batch_size(val_loader, ratio=0.5)
691
+
692
+ logging_dict = {'lr': [group['lr'] for group in optimizer.param_groups], 'train_loss': float('inf')}
693
+ else:
694
+ raise
695
+
696
+ training_log[epoch] = logging_dict
697
+ if self.is_early_stopping(epoch):
698
+ print(f'[Trainer CallBack] 📢 Epoch {epoch}/{epochs}: Detect Overfitting! Breaking Training Process...')
699
+ break
700
+ if self.logging:
701
+ if epoch % self.logging == 0:
702
+ print(f'[Trainer CallBack] 📢 Epoch {epoch}/{epochs}:', fmt(logging_dict))
703
+ else:
704
+ print(f'{epoch}...', end=' ')
705
+
706
+ if self._at_time_limit(start_training_time):
707
+ print(f'[Trainer CallBack] ⚠️ Epoch {epoch}/{epochs}: Thời gian training giới hạn là {self.training_time}, hết giờ tại epoch {epoch}/{epochs}')
708
+ break
709
+
710
+ if self.logging_file:
711
+ os.makedirs(f'{self.checkpoints_dir}/logs', exist_ok=True)
712
+ with open(f"{self.checkpoints_dir}/logs/{self.save_name}_logging.json", "a", encoding="utf-8") as f:
713
+ f.write(json.dumps(training_log))
714
+
715
+ if self.use_ema and self.ema_net is not None:
716
+ self._save_state_dict(self.ema_net.module)
717
+ else:
718
+ self._save_state_dict(network)
719
+ print(f'[Trainer CallBack] 📢 Kết thúc training.\n')
720
+
721
+ best_model, last_model = None, None
722
+ eval_net = self.ema_net.module if (self.use_ema and self.ema_net is not None) else network
723
+ if self.return_best :
724
+ best_model = self.best_stage[0][2] if self.best_stage[0][2] is not None else eval_net.state_dict()
725
+ best_model = {k.replace("module.", ""): v.detach().cpu().clone() for k, v in best_model.items()}
726
+ if self.return_last:
727
+ last_model = eval_net.state_dict()
728
+ last_model = {k.replace("module.", ""): v.detach().cpu().clone() for k, v in last_model.items()}
729
+
730
+ del network
731
+ torch.cuda.empty_cache()
732
+ gc.collect()
733
+ return training_log, best_model, last_model
734
+
735
+ def _time_str_to_seconds(self, time_str):
736
+ days, hours, minutes, seconds = map(int, time_str.split(":"))
737
+ return days * 86400 + hours * 3600 + minutes * 60 + seconds
738
+
739
+ def _update_best_network(self, network, val_score, epoch):
740
+ topk = max(1, self.topk)
741
+ self.best_stage.append([val_score, epoch, {k: v.detach().cpu().clone() for k, v in network.state_dict().items()}])
742
+ self.best_stage = sorted(self.best_stage, reverse=(self.mode == 'max'), key=lambda x: x[0])[:topk]
743
+ if val_score in [x[0] for x in self.best_stage]:
744
+ return True
745
+ return False
746
+
747
+ def is_early_stopping(self, epoch):
748
+ if self.best_stage[0][1] is None:
749
+ return False
750
+ if not self.early_stopping:
751
+ return False
752
+ return epoch - self.best_stage[0][1] >= self.early_stopping
753
+
754
+ def _at_time_limit(self, start_training_time):
755
+ return time.time() - start_training_time >= self.training_time
756
+
757
+ def _save_state_dict(self, network):
758
+ if self.topk <= 0:
759
+ return
760
+
761
+ if self.save_best:
762
+ for r in range(self.topk):
763
+ os.makedirs(f'{self.checkpoints_dir}/r{r+1}s', exist_ok=True)
764
+
765
+ for rank, (score, epoch, state_dict) in enumerate(self.best_stage):
766
+ if state_dict is None:
767
+ continue
768
+ state_dict = {k.replace("module.", ""): v.detach().cpu().clone() for k, v in state_dict.items()}
769
+ torch.save(state_dict, f'{self.checkpoints_dir}/r{rank+1}s/{self.save_name}_r{rank+1}_vs{score:.5f}_{"ema" if self.ema_net is not None else ""}.pth')
770
+ if self.save_last:
771
+ os.makedirs(f'{self.checkpoints_dir}/lasts', exist_ok=True)
772
+ state_dict = {k.replace("module.", ""): v.detach().cpu().clone() for k, v in network.state_dict().items()}
773
+ torch.save(state_dict, f'{self.checkpoints_dir}/lasts/{self.save_name}_last_{"ema" if self.ema_net is not None else ""}.pth')
774
+
775
+ def _train_epoch(self, network, train_loader, optimizer, scheduler, loss_fn, encoded_docs):
776
+ network.train()
777
+ total_loss = 0
778
+ total_loss_dict = {}
779
+ for batch_idx, batch in enumerate(train_loader):
780
+ optimizer.zero_grad()
781
+ with torch.autocast(device_type=self.device, dtype=torch.float16):
782
+ loss, loss_dict = self._cal_loss(network, batch, batch_idx, loss_fn, encoded_docs)
783
+
784
+ for k, v in loss_dict.items():
785
+ t = total_loss_dict.get(k, 0)
786
+ total_loss_dict[k] = t + v
787
+ self.grad_scaler.scale(loss).backward()
788
+ self.grad_scaler.unscale_(optimizer)
789
+ grad_norm = nn.utils.clip_grad_norm_(network.parameters(), self.max_grad_norm)
790
+ # print(grad_norm) # Bỏ cmt dòng này để biết nên chọn max_grad_norm bằng bao nhiêu...
791
+ self.grad_scaler.step(optimizer)
792
+ self.grad_scaler.update()
793
+ if self.schedule_in_step and scheduler:
794
+ scheduler.step()
795
+ if self.use_ema and self.ema_net is not None:
796
+ self.ema_net.update(network)
797
+ total_loss += loss
798
+ return (total_loss / len(train_loader)).item(), {k: v.item() / len(train_loader) for k, v in total_loss_dict.items()}
799
+
800
+ def _eval_epoch(self, network, val_loader, eval_fn, encoded_docs):
801
+ network.eval()
802
+ total_score = 0.0
803
+ total_score_dict = {}
804
+ object_lists = None # sẽ init sau
805
+
806
+ with torch.no_grad():
807
+ for batch_idx, batch in enumerate(val_loader):
808
+ score, score_dict, objects = self._cal_val_score(network, batch, batch_idx, eval_fn, encoded_docs)
809
+ total_score += score
810
+
811
+ for k, v in score_dict.items():
812
+ t = total_score_dict.get(k, 0)
813
+ total_score_dict[k] = t + v
814
+
815
+ if objects:
816
+ if object_lists is None:
817
+ object_lists = [[] for _ in range(len(objects))]
818
+
819
+ for i, obj in enumerate(objects):
820
+ object_lists[i].append(obj.detach())
821
+
822
+ if object_lists is not None:
823
+ object_arrays = [
824
+ torch.concat(obj_list, dim=0).cpu().numpy()
825
+ for obj_list in object_lists
826
+ ]
827
+ else:
828
+ object_arrays = []
829
+
830
+ return total_score / len(val_loader), {k: v / len(val_loader) for k, v in total_score_dict.items()}, object_arrays
831
+
832
+ def _get_encoded_docs(self, network, corpus_loader):
833
+ network.eval()
834
+ with torch.no_grad():
835
+ encoded_docs = []
836
+ for batch_idx, batch in enumerate(corpus_loader):
837
+ input_ids = batch['input_ids'].to(self.device)
838
+ attn_mask = batch['attn_mask'].to(self.device)
839
+ encoded_doc = network(input_ids, attn_mask, is_query=False)
840
+ encoded_docs.append(encoded_doc)
841
+ encoded_docs = torch.concat(encoded_docs, dim=0).squeeze(1)
842
+ return encoded_docs
843
+
844
+ def _cal_loss(self, network, batch, batch_idx, loss_fn, encoded_docs):
845
+ # Bạn cần override _cal_loss để tính loss
846
+ text_input_ids = batch['text_input_ids'].to(self.device)
847
+ text_attn_mask = batch['text_attn_mask'].to(self.device)
848
+ pos_idxes = batch['pos_idxes'].to(self.device)
849
+ pos_mask = batch['pos_mask'].to(self.device)
850
+ neg_idxes = batch['neg_idxes'].to(self.device)
851
+
852
+ encoded_text = network(text_input_ids, text_attn_mask, is_query=True)
853
+ encoded_pos = encoded_docs[pos_idxes]
854
+ encoded_neg = encoded_docs[neg_idxes]
855
+
856
+ loss_dict = loss_fn(encoded_text, encoded_pos, encoded_neg, pos_mask)
857
+ return loss_dict['total'], loss_dict
858
+
859
+ def _cal_val_score(self, network, batch, batch_idx, eval_fn, encoded_docs):
860
+ # Bạn cần override _cal_val_score để tính val score, list bên cạnh là để trả về y hay pred gì đó (nếu cần)
861
+ text_input_ids = batch['text_input_ids'].to(self.device)
862
+ text_attn_mask = batch['text_attn_mask'].to(self.device)
863
+ gt_pos_idxes = batch['gt_pos_idxes']
864
+ encoded_text = network(text_input_ids, text_attn_mask, is_query=True)
865
+
866
+ scores = torch.matmul(encoded_text, encoded_docs.T)
867
+ topk_scores, topk_indices = torch.topk(scores, k=10)
868
+ pred_topk = [
869
+ idx[score > 0].tolist()
870
+ for score, idx in zip(topk_scores, topk_indices)
871
+ ]
872
+
873
+ pred_topk = list_to_tuple(pred_topk)
874
+ gt_pos_idxes = list_to_tuple(gt_pos_idxes)
875
+ score_dict = eval_fn(pred_topk, gt_pos_idxes)
876
+ return score_dict['recall'], score_dict, []
877
+
878
+ # %% [code]
879
+ def tokenize_to_parts(text, tokenizer, max_len, max_n_parts):
880
+ # Tokenize với overflow để chia thành nhiều đoạn
881
+ enc = tokenizer(
882
+ text,
883
+ max_length=max_len*max_n_parts,
884
+ truncation=True,
885
+ padding="max_length",
886
+ return_overflowing_tokens=True,
887
+ return_tensors="pt"
888
+ )
889
+
890
+ input_ids = enc["input_ids"].reshape(max_n_parts, max_len) # (n_parts, max_len)
891
+ attn_mask = enc["attention_mask"].reshape(max_n_parts, max_len) # (n_parts, max_len)
892
+
893
+ return input_ids, attn_mask
894
+
895
+ class LawRetrievalDataset(Dataset):
896
+ def __init__(self, all_data, using_idxes, corpus_dict, tokenizer, max_len, max_n_parts, n_negs):
897
+ super().__init__()
898
+
899
+ self.all_data = all_data
900
+ self.using_idxes = using_idxes
901
+ self.tokenizer = tokenizer
902
+ self.max_len = max_len
903
+ self.max_n_parts = max_n_parts
904
+ self.n_negs = n_negs
905
+
906
+ # ===== BUILD CORPUS =====
907
+ idx = 0
908
+ self.corpus_list = []
909
+ self.corpus_dict = {}
910
+
911
+ for doc_name, articles_dict in corpus_dict.items():
912
+ self.corpus_dict[doc_name] = {}
913
+ for article_idx, content in articles_dict.items():
914
+ self.corpus_list.append([doc_name, article_idx, content])
915
+ self.corpus_dict[doc_name][article_idx] = {
916
+ 'content': content,
917
+ 'idx': idx
918
+ }
919
+ idx += 1
920
+
921
+ def __len__(self):
922
+ return len(self.using_idxes)
923
+
924
+ # ===== ENCODE DOC =====
925
+ def _encode_contexts(self, idxes):
926
+ all_input_ids, all_attn_mask = [], []
927
+
928
+ for idx in idxes:
929
+ name, art, _ = self.corpus_list[idx]
930
+ corpus = self.corpus_dict[name][art]
931
+
932
+ if 'content_input_ids' in corpus:
933
+ content_input_ids = corpus['content_input_ids']
934
+ content_attn_mask = corpus['content_attn_mask']
935
+ else:
936
+ content = corpus['content']
937
+ content_input_ids, content_attn_mask = tokenize_to_parts(
938
+ content, self.tokenizer, self.max_len, self.max_n_parts
939
+ )
940
+ corpus['content_input_ids'] = content_input_ids
941
+ corpus['content_attn_mask'] = content_attn_mask
942
+
943
+ all_input_ids.append(content_input_ids)
944
+ all_attn_mask.append(content_attn_mask)
945
+
946
+ all_input_ids = torch.stack(all_input_ids)
947
+ all_attn_mask = torch.stack(all_attn_mask)
948
+
949
+ return all_input_ids, all_attn_mask
950
+
951
+ def __getitem__(self, idx):
952
+ ridx = self.using_idxes[idx]
953
+ data = self.all_data[ridx]
954
+
955
+ query_text = data['text']
956
+
957
+ text_input_ids, text_attn_mask = tokenize_to_parts(
958
+ query_text, self.tokenizer, self.max_len, 1
959
+ )
960
+
961
+ # ===== POS =====
962
+ gt_pos_idxes = []
963
+ hard_names = []
964
+ for law in data['relevant_law']:
965
+ name = law['doc']
966
+ art = law['art']
967
+ gt_pos_idxes.append(self.corpus_dict[name][art]['idx'])
968
+ if name not in hard_names:
969
+ hard_names.append(name)
970
+
971
+ pos_idxes = torch.tensor(gt_pos_idxes, dtype=torch.long)
972
+ pos_mask = torch.ones(len(pos_idxes))
973
+
974
+ # ===== NEG =====
975
+ hard_neg_idxes = []
976
+ for name in hard_names:
977
+ for content in self.corpus_dict[name].values():
978
+ if content['idx'] in gt_pos_idxes:
979
+ continue
980
+ hard_neg_idxes.append(content['idx'])
981
+
982
+ easy_neg_idxes = list(range(len(self.corpus_list)))
983
+ for i in gt_pos_idxes + hard_neg_idxes:
984
+ if i in easy_neg_idxes:
985
+ easy_neg_idxes.remove(i)
986
+
987
+ n_hards = min(len(hard_neg_idxes), self.n_negs // 2)
988
+ neg_idxes = random.sample(hard_neg_idxes, n_hards) + random.sample(easy_neg_idxes, self.n_negs - n_hards)
989
+ neg_idxes = torch.tensor(neg_idxes, dtype=torch.long)
990
+
991
+ return {
992
+ 'text_input_ids': text_input_ids,
993
+ 'text_attn_mask': text_attn_mask,
994
+ 'gt_pos_idxes': gt_pos_idxes,
995
+ 'pos_idxes': pos_idxes,
996
+ 'pos_mask': pos_mask,
997
+ 'neg_idxes': neg_idxes,
998
+ }
999
+
1000
+ class CorpusDataset(Dataset):
1001
+ def __init__(self, corpus_dict, tokenizer, max_len, max_n_parts):
1002
+ super().__init__()
1003
+ self.tokenizer = tokenizer
1004
+ self.max_len = max_len
1005
+ self.max_n_parts = max_n_parts
1006
+
1007
+ idx = 0
1008
+ self.corpus_list = []
1009
+ self.corpus_dict = {}
1010
+ for doc_name, articles_dict in corpus_dict.items():
1011
+ self.corpus_dict[doc_name] = {}
1012
+ for article_idx, content in articles_dict.items():
1013
+ self.corpus_list.append([doc_name, article_idx, content])
1014
+ self.corpus_dict[doc_name][article_idx] = {'content': content, 'idx': idx}
1015
+ idx += 1
1016
+
1017
+ def __len__(self):
1018
+ return len(self.corpus_list)
1019
+
1020
+ def _encode_contexts(self, idxes):
1021
+ all_input_ids, all_attn_mask = [], []
1022
+ for idx in idxes:
1023
+ name = self.corpus_list[idx][0]
1024
+ art = self.corpus_list[idx][1]
1025
+ corpus = self.corpus_dict[name][art]
1026
+ if 'content_input_ids' in corpus and 'content_attn_mask' in corpus:
1027
+ content_input_ids = corpus['content_input_ids']
1028
+ content_attn_mask = corpus['content_attn_mask']
1029
+ else:
1030
+ content = corpus['content']
1031
+ content_input_ids, content_attn_mask = tokenize_to_parts(content, self.tokenizer, self.max_len, self.max_n_parts)
1032
+ corpus['content_input_ids'] = content_input_ids
1033
+ corpus['content_attn_mask'] = content_attn_mask
1034
+
1035
+ all_input_ids.append(content_input_ids)
1036
+ all_attn_mask.append(content_attn_mask)
1037
+
1038
+ all_input_ids = torch.stack(all_input_ids)
1039
+ all_attn_mask = torch.stack(all_attn_mask)
1040
+ return all_input_ids, all_attn_mask
1041
+
1042
+ def __getitem__(self, idx):
1043
+ input_ids, attn_mask = self._encode_contexts([idx])
1044
+
1045
+ return {
1046
+ 'input_ids': input_ids,
1047
+ 'attn_mask': attn_mask,
1048
+ }
1049
+
1050
+ def _pad_batch(tensor_list, pad_value=0):
1051
+ """
1052
+ tensor_list: list of tensors, mỗi tensor shape (Nk, max_n_parts, max_len)
1053
+ return: tensor shape (B, max_Nk, max_n_parts, max_len)
1054
+ """
1055
+ max_Nk = max(t.size(0) for t in tensor_list)
1056
+
1057
+ padded = []
1058
+ for t in tensor_list:
1059
+ Nk = t.size(0)
1060
+
1061
+ if Nk < max_Nk:
1062
+ pad_shape = (max_Nk - Nk, *t.shape[1:])
1063
+ pad_tensor = t.new_full(pad_shape, pad_value)
1064
+ t = torch.cat([t, pad_tensor], dim=0)
1065
+
1066
+ padded.append(t)
1067
+
1068
+ return torch.stack(padded) # (B, max_Nk, max_n_parts, max_len)
1069
+
1070
+ def collate_fn(batch):
1071
+ text_input_ids = torch.stack([b["text_input_ids"] for b in batch])
1072
+ text_attn_mask = torch.stack([b["text_attn_mask"] for b in batch])
1073
+ gt_pos_idxes = [b["gt_pos_idxes"] for b in batch]
1074
+ neg_idxes = torch.stack([b["neg_idxes"] for b in batch])
1075
+
1076
+ pos_idxes = [b["pos_idxes"].unsqueeze(-1).unsqueeze(-1) for b in batch]
1077
+ pos_mask = [b["pos_mask"].unsqueeze(-1).unsqueeze(-1) for b in batch]
1078
+
1079
+ # pad theo Nk
1080
+ pos_idxes = _pad_batch(pos_idxes, pad_value=0).squeeze(-1).squeeze(-1)
1081
+ pos_mask = _pad_batch(pos_mask, pad_value=0).squeeze(-1).squeeze(-1)
1082
+
1083
+ return {
1084
+ 'text_input_ids': text_input_ids,
1085
+ 'text_attn_mask': text_attn_mask,
1086
+ 'gt_pos_idxes': gt_pos_idxes,
1087
+ 'pos_idxes': pos_idxes,
1088
+ 'pos_mask': pos_mask,
1089
+ 'neg_idxes': neg_idxes,
1090
+ }
1091
+
1092
+ # %% [code]
1093
+ def encode_corpus(state_dicts, network, corpus_loader, device):
1094
+ if torch.cuda.device_count() > 1:
1095
+ network = nn.DataParallel(network)
1096
+ network.to(device)
1097
+ network.eval()
1098
+
1099
+ all_model_embs = []
1100
+ for i, state_dict in enumerate(state_dicts):
1101
+ # ===== load model =====
1102
+ if torch.cuda.device_count() > 1:
1103
+ network.module.load_state_dict(state_dict)
1104
+ else:
1105
+ network.load_state_dict(state_dict)
1106
+
1107
+ encoded_docs = []
1108
+
1109
+ with torch.no_grad():
1110
+ for batch in corpus_loader:
1111
+ input_ids = batch['input_ids'].to(device)
1112
+ attn_mask = batch['attn_mask'].to(device)
1113
+
1114
+ emb = network(input_ids, attn_mask, is_query=False) # [B, 1, D] hoặc [B, D]
1115
+
1116
+ encoded_docs.append(emb)
1117
+
1118
+ encoded_docs = torch.concat(encoded_docs, dim=0).squeeze(1) # [N, D]
1119
+ all_model_embs.append(encoded_docs)
1120
+
1121
+ # ===== ensemble =====
1122
+ # stack → [M, N, D]
1123
+ all_model_embs = torch.stack(all_model_embs, dim=0)
1124
+ final_embs = all_model_embs.mean(dim=0) # [N, D]
1125
+
1126
+ return final_embs
1127
+
1128
+ def test(state_dicts, network, test_loader, device, eval_fn, encoded_docs, bm25_scores, topks=[5, 10, 15]):
1129
+ if torch.cuda.device_count() > 1:
1130
+ network = nn.DataParallel(network)
1131
+ network.to(device)
1132
+ network.eval()
1133
+
1134
+ per_model_scores = []
1135
+ max_k = max(topks)
1136
+
1137
+ all_scores = []
1138
+ all_gt_pos_idxes = []
1139
+ with torch.no_grad():
1140
+ for batch in test_loader:
1141
+ text_input_ids = batch['text_input_ids'].to(device)
1142
+ text_attn_mask = batch['text_attn_mask'].to(device)
1143
+ gt_pos_idxes = batch['gt_pos_idxes']
1144
+ all_gt_pos_idxes.extend(gt_pos_idxes)
1145
+
1146
+ list_encoded_texts = []
1147
+
1148
+ for state_dict in state_dicts:
1149
+ # ===== load model =====
1150
+ if torch.cuda.device_count() > 1:
1151
+ network.module.load_state_dict(state_dict)
1152
+ else:
1153
+ network.load_state_dict(state_dict)
1154
+
1155
+ encoded_text = network(text_input_ids, text_attn_mask, is_query=True)
1156
+ list_encoded_texts.append(encoded_text)
1157
+
1158
+ ensemble_encoded_text = torch.stack(list_encoded_texts, dim=0).mean(dim=0)
1159
+ scores = torch.matmul(ensemble_encoded_text, encoded_docs.T) # B, M
1160
+ all_scores.append(scores)
1161
+
1162
+ all_scores = torch.concat(all_scores, dim=0) # (N, M)
1163
+
1164
+ bm25_scores = torch.tensor(bm25_scores).to(all_scores.device) # (N, M)
1165
+ bm25_scores = bm25_scores.float()
1166
+
1167
+ # min-max về [0, 1]
1168
+ bm25_scores = (
1169
+ bm25_scores - bm25_scores.min(dim=-1, keepdim=True).values
1170
+ ) / (
1171
+ bm25_scores.max(dim=-1, keepdim=True).values
1172
+ - bm25_scores.min(dim=-1, keepdim=True).values
1173
+ + 1e-8
1174
+ )
1175
+ # đổi sang [-1, 1]
1176
+ bm25_scores = bm25_scores * 2 - 1
1177
+
1178
+ all_gt_pos_idxes = list_to_tuple(all_gt_pos_idxes)
1179
+
1180
+ final_score = {}
1181
+ for weight in [0, 0.25, 0.5, 0.75, 1]:
1182
+
1183
+ # score cuối
1184
+ final_scores = all_scores + weight * bm25_scores
1185
+
1186
+ # topk lớn nhất
1187
+ topk_scores, topk_indices = torch.topk(final_scores, k=max_k)
1188
+
1189
+ pred_topk_full = [
1190
+ idx[score > 0].tolist()
1191
+ for score, idx in zip(topk_scores, topk_indices)
1192
+ ]
1193
+
1194
+ pred_topk_full = list_to_tuple(pred_topk_full)
1195
+
1196
+ final_score[weight] = {}
1197
+
1198
+ for k in topks:
1199
+ pred_topk_k = [p[:k] for p in pred_topk_full]
1200
+ final_score[weight][k] = eval_fn(pred_topk_k, all_gt_pos_idxes)
1201
+
1202
+ return final_score
1203
+
1204
+ # %% [code]
1205
+ with open(f'{train_dir}/train.json', "r", encoding="utf-8") as f:
1206
+ data_train = json.load(f)
1207
+
1208
+ with open(f'{test_dir}/test.json', "r", encoding="utf-8") as f:
1209
+ data_test = json.load(f)
1210
+
1211
+ with open(f'{test_dir}/corpus.json', "r", encoding="utf-8") as f:
1212
+ data_corpus = json.load(f)
1213
+
1214
+ print('Train:', len(data_train))
1215
+ print('Test:', len(data_test))
1216
+ print('Corpus:', len(data_corpus))
1217
+
1218
+ # %% [code]
1219
+ # trigger_types = sorted(list(set([e['label'] for d in data_train + data_test for e in d['issues']]))) # NBR : Neighbor relation
1220
+ # bio_trigger_types = ['O'] + [f'{prefix}-{trg}' for trg in trigger_types for prefix in ['B', 'I']]
1221
+ # trigger_label2id = {l: i for i, l in enumerate(bio_trigger_types)}
1222
+ # trigger_id2label = {i: l for l, i in trigger_label2id.items()}
1223
+
1224
+ # argument_types = sorted(list(set([a['role'] for d in data_train + data_test for e in d['issues'] for a in e['arguments']])))
1225
+ # bio_argument_types = ['O'] + [f'{prefix}-{arg}' for arg in argument_types for prefix in ['B', 'I']]
1226
+ # argument_label2id = {l: i for i, l in enumerate(bio_argument_types)}
1227
+ # argument_id2label = {i: l for l, i in argument_label2id.items()}
1228
+
1229
+ # label2id = {
1230
+ # 'Trg': trigger_label2id,
1231
+ # 'Arg': argument_label2id,
1232
+ # }
1233
+
1234
+ # id2label = {
1235
+ # 'Trg': trigger_id2label,
1236
+ # 'Arg': argument_id2label,
1237
+ # }
1238
+
1239
+ # %% [code]
1240
+ # zero_events_idxes = []
1241
+ # for idx, d in enumerate(data_train):
1242
+ # if len(d['issues']) == 0:
1243
+ # zero_events_idxes.append(idx)
1244
+
1245
+ # n_zero_events_samples = len(zero_events_idxes)
1246
+ # n_has_events_samples = len(data_train) - n_zero_events_samples
1247
+
1248
+ # random.seed(42)
1249
+ # k = min(int(n_has_events_samples * zero_events_rate), len(zero_events_idxes))
1250
+ # sampled_zero_events_idxes = random.sample(zero_events_idxes, k)
1251
+
1252
+ # new_data_train = []
1253
+ # for idx, d in enumerate(data_train):
1254
+ # if len(d['issues']) == 0:
1255
+ # if idx in sampled_zero_events_idxes:
1256
+ # new_data_train.append(d)
1257
+ # else:
1258
+ # new_data_train.append(d)
1259
+ # data_train = new_data_train
1260
+
1261
+ # print('Train:', len(data_train))
1262
+
1263
+ # %% [code]
1264
+ if debug_only:
1265
+ data_train = data_train[:20]
1266
+ data_test = data_test[:20]
1267
+
1268
+ print('Train:', len(data_train))
1269
+ print('Test:', len(data_test))
1270
+
1271
+ # %% [code]
1272
+ tokenizer = AutoTokenizer.from_pretrained(backbone_model_name)
1273
+
1274
+ # %% [code]
1275
+ print('Experiment name:', state_dict_save_name)
1276
+
1277
+ # %% [code]
1278
+ if not test_only:
1279
+ full_idxes = np.array(range(len(data_train)))
1280
+ training_logs, best_models, last_models = [], [], []
1281
+ start_training_time = time.time()
1282
+ for seed in SEEDS:
1283
+ kf = KFold(n_splits=nfolds, shuffle=True, random_state=seed)
1284
+ generator = torch.Generator()
1285
+ generator.manual_seed(seed)
1286
+
1287
+ corpusset = CorpusDataset(data_corpus, tokenizer, **corpus_memory_params)
1288
+ corpus_loader = DataLoader(corpusset, generator=generator, **val_loader_params)
1289
+ for fold_idx, (tr_idx, va_idx) in enumerate(kf.split(full_idxes)):
1290
+ if only_fold_idx is not None and only_fold_idx >= 0 and only_fold_idx != fold_idx:
1291
+ continue
1292
+ set_seed(seed)
1293
+
1294
+ train_idxes, val_idxes = full_idxes[tr_idx], full_idxes[va_idx]
1295
+
1296
+ trainset = LawRetrievalDataset(data_train, train_idxes, data_corpus, tokenizer, **train_memory_params)
1297
+ valset = LawRetrievalDataset(data_train, val_idxes, data_corpus, tokenizer, **val_memory_params)
1298
+
1299
+ train_loader = DataLoader(trainset, generator=generator, collate_fn=collate_fn, **train_loader_params)
1300
+ val_loader = DataLoader(valset, generator=generator, collate_fn=collate_fn, **val_loader_params)
1301
+
1302
+ my_model = EncodeModel(
1303
+ **model_params
1304
+ )
1305
+ total_params = sum(p.numel() for p in my_model.parameters())
1306
+ print(f"Total params: {total_params:,}")
1307
+
1308
+ # optimizer, scheduler = configure_optimizers(my_model, optim_params, scheduler_params)
1309
+ encoder_params = set(map(id, my_model.encoder.parameters()))
1310
+ other_params = [
1311
+ p for p in my_model.parameters()
1312
+ if id(p) not in encoder_params
1313
+ ]
1314
+ optimizer = optim.AdamW([
1315
+ {"params": my_model.encoder.parameters(), "lr": 2e-5},
1316
+ {"params": other_params}
1317
+ ], lr=5e-4)
1318
+ scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=20, eta_min=1e-6)
1319
+
1320
+ loss_fn = CustomLoss(
1321
+ **loss_func_params
1322
+ )
1323
+ eval_fn = CustomEvalFn(**eval_func_params)
1324
+ trainer_params['save_name'] = f'{state_dict_save_name}_s{seed}_f{fold_idx}'
1325
+ trainer = Trainer(**trainer_params)
1326
+
1327
+ print(f'Start Training Fold {fold_idx}...')
1328
+ training_log, best_model, last_model = trainer.fit(
1329
+ my_model, optimizer, scheduler, loss_fn, epochs, train_loader, val_loader, corpus_loader, eval_fn,
1330
+ start_epoch=1, start_training_time=start_training_time, refresh_every=2,
1331
+ )
1332
+
1333
+ training_logs.append(training_log)
1334
+ best_models.append(best_model)
1335
+ last_models.append(last_model)
1336
+
1337
+ # %% [code]
1338
+ def load_all_state_dicts(folder):
1339
+ files = []
1340
+
1341
+ for file in os.listdir(folder):
1342
+ if file.endswith(".pt") or file.endswith(".pth"):
1343
+ m = re.search(r"f(\d+)", file) # tìm f<số>
1344
+ if m:
1345
+ fold = int(m.group(1))
1346
+ files.append((fold, file))
1347
+
1348
+ # sort theo fold
1349
+ files.sort(key=lambda x: x[0])
1350
+
1351
+ state_dicts = []
1352
+ for fold, file in files:
1353
+ path = os.path.join(folder, file)
1354
+ print(f"Loading fold {fold}: {file}")
1355
+ state_dict = torch.load(path, map_location="cpu")
1356
+ state_dicts.append(state_dict)
1357
+
1358
+ return state_dicts
1359
+
1360
+ if test_only:
1361
+ snapshot_download(repo_id=repo_name, local_dir="", repo_type="model", allow_patterns=[f"{state_dict_save_name}/**"])
1362
+ get_ipython().system('rm -rf .cache .gitattributes')
1363
+
1364
+ best_models = load_all_state_dicts(f"{state_dict_save_name}/r1s")
1365
+ last_models = load_all_state_dicts(f"{state_dict_save_name}/lasts")
1366
+
1367
+ # %% [code]
1368
+ os.makedirs(f'{checkpoints_dir}/results', exist_ok=True)
1369
+ testset = LawRetrievalDataset(data_test, range(len(data_test)), data_corpus, tokenizer, **val_memory_params)
1370
+ generator = torch.Generator()
1371
+ test_loader = DataLoader(testset, generator=generator, collate_fn=collate_fn, **val_loader_params)
1372
+ eval_fn = CustomEvalFn(**eval_func_params)
1373
+ my_model = EncodeModel(
1374
+ **model_params
1375
+ )
1376
+ total_params = sum(p.numel() for p in my_model.parameters())
1377
+ print(f"Total params: {total_params:,}")
1378
+
1379
+ # %% [code]
1380
+ start_time = time.time()
1381
+ encoded_docs = encode_corpus(best_models, my_model, corpus_loader, device)
1382
+ best_score = test(best_models, my_model, test_loader, device, eval_fn, encoded_docs, bm25_scores)
1383
+
1384
+ encoded_docs = encode_corpus(last_models, my_model, corpus_loader, device)
1385
+ last_score = test(last_models, my_model, test_loader, device, eval_fn, encoded_docs, bm25_scores)
1386
+
1387
+ result_test = {"Best model": best_score, "Last model": last_score}
1388
+
1389
+ with open(f"{checkpoints_dir}/results/{state_dict_save_name}_test.json", "w", encoding="utf-8") as f:
1390
+ json.dump(result_test, f, ensure_ascii=False, indent=2)
1391
+
1392
+ print('Test:', time.time() - start_time, 's --> Done!')
1393
+
1394
+ # %% [code]
1395
+ def dict_to_df(data):
1396
+ """
1397
+ data format:
1398
+ {
1399
+ model_name: {
1400
+ weight: {
1401
+ topk: {
1402
+ metric: value
1403
+ }
1404
+ }
1405
+ }
1406
+ }
1407
+ """
1408
+
1409
+ row_tuples = []
1410
+ row_values = []
1411
+
1412
+ # ===== lấy model đầu tiên =====
1413
+ first_model = next(iter(data.values()))
1414
+
1415
+ # ===== weight keys =====
1416
+ weight_keys = list(first_model.keys())
1417
+
1418
+ # ===== topk keys =====
1419
+ first_weight = next(iter(first_model.values()))
1420
+ topk_keys = list(first_weight.keys())
1421
+
1422
+ # ===== metric keys =====
1423
+ first_topk = next(iter(first_weight.values()))
1424
+ metrics = list(first_topk.keys())
1425
+
1426
+ for weight in weight_keys:
1427
+
1428
+ for topk in topk_keys:
1429
+
1430
+ # ===== multi index row =====
1431
+ row_tuples.append((weight, topk))
1432
+
1433
+ row = {}
1434
+
1435
+ for model_name, model_data in data.items():
1436
+
1437
+ for metric in metrics:
1438
+
1439
+ row[(model_name, metric)] = (
1440
+ model_data[weight][topk][metric]
1441
+ )
1442
+
1443
+ row_values.append(row)
1444
+
1445
+ # ===== dataframe =====
1446
+ df = pd.DataFrame(row_values)
1447
+
1448
+ # ===== multi columns =====
1449
+ df.columns = pd.MultiIndex.from_tuples(
1450
+ df.columns,
1451
+ names=["model", "metric"]
1452
+ )
1453
+
1454
+ # ===== multi index =====
1455
+ df.index = pd.MultiIndex.from_tuples(
1456
+ row_tuples,
1457
+ names=["weight", "topk"]
1458
+ )
1459
+
1460
+ return df
1461
+
1462
+ result_test_df = dict_to_df(result_test)
1463
+ result_test_df.to_excel(f"{checkpoints_dir}/results/{state_dict_save_name}_test_df.xlsx")
1464
+ result_test_df
1465
+
1466
+ # %% [code]
1467
+ key = ("Best model", "recall")
1468
+
1469
+ result_test_df_best = (
1470
+ result_test_df
1471
+ .sort_values(by=key, ascending=False)
1472
+ .groupby(level="weight")
1473
+ .head(1)
1474
+ )
1475
+ result_test_df_best.to_excel(f"{checkpoints_dir}/results/{state_dict_save_name}_test_df_best.xlsx")
1476
+ result_test_df_best
1477
+
1478
+ # %% [code]
1479
+ def get_avg_best_score(logs):
1480
+ return float(np.mean([list(log.values())[-1]['best_score'] for log in logs]))
1481
+
1482
+ def get_avg_log(logs, epochs):
1483
+ avg_log = {}
1484
+
1485
+ for epoch in range(1, epochs + 1):
1486
+ val_score = 0.0
1487
+ train_loss = 0.0
1488
+ n_eval = 0
1489
+
1490
+ for idx in range(len(logs)):
1491
+ log = logs[idx].get(epoch, logs[idx].get(str(epoch)))
1492
+ if log is None:
1493
+ continue
1494
+
1495
+ val_score += log.get('val_score', 0.0)
1496
+ train_loss += log.get('train_loss', 0.0)
1497
+ n_eval += 1
1498
+
1499
+ if n_eval == 0:
1500
+ continue
1501
+
1502
+ avg_log[epoch] = {
1503
+ 'train_loss': train_loss / n_eval,
1504
+ 'val_score': val_score / n_eval if val_score != 0 else float('inf')
1505
+ }
1506
+
1507
+ return avg_log
1508
+
1509
+ def parse_label_key(label: str):
1510
+ try:
1511
+ first = float(label.split('_', 1)[0]) # số đầu: trước dấu _
1512
+ last = float(re.findall(r'_(\d+(?:\.\d+)?)$', label)[0])
1513
+ return first, last
1514
+ except:
1515
+ return (0, 0)
1516
+
1517
+ def plot_training_logs(logs_dict, save_path=None, figsize=(24, 10)):
1518
+ fig, axes = plt.subplots(1, 2, figsize=figsize)
1519
+
1520
+ # ===== Plot Train Loss =====
1521
+ for name, log in logs_dict.items():
1522
+ epochs = sorted(log.keys())
1523
+ train_loss = [log[e]['train_loss'] for e in epochs]
1524
+ axes[0].plot(epochs, train_loss, label=name)
1525
+
1526
+ axes[0].set_xlabel('Epoch')
1527
+ axes[0].set_ylabel('Train Loss')
1528
+ axes[0].set_title('Training Loss')
1529
+ axes[0].grid(True)
1530
+
1531
+ # ===== Plot Validation Score =====
1532
+ for name, log in logs_dict.items():
1533
+ epochs = sorted(log.keys())
1534
+ val_score = [log[e]['val_score'] for e in epochs]
1535
+ axes[1].plot(epochs, val_score, label=name)
1536
+
1537
+ axes[1].set_xlabel('Epoch')
1538
+ axes[1].set_ylabel('Validation Score')
1539
+ axes[1].set_title('Validation Score')
1540
+ axes[1].grid(True)
1541
+
1542
+ # ===== Shared Legend =====
1543
+ handles, labels = axes[0].get_legend_handles_labels()
1544
+ pairs = list(zip(handles, labels))
1545
+ pairs_sorted = sorted(
1546
+ pairs,
1547
+ key=lambda x: parse_label_key(x[1])
1548
+ )
1549
+ handles_sorted, labels_sorted = zip(*pairs_sorted)
1550
+
1551
+ axes[0].legend(
1552
+ handles_sorted,
1553
+ labels_sorted,
1554
+ loc='center left',
1555
+ bbox_to_anchor=(1.01, 0.5),
1556
+ borderaxespad=0.
1557
+ )
1558
+
1559
+ plt.tight_layout(rect=[0, 0, 1, 1])
1560
+
1561
+ if save_path is not None:
1562
+ os.makedirs(os.path.dirname(save_path), exist_ok=True) if os.path.dirname(save_path) else None
1563
+ plt.savefig(save_path, dpi=300, bbox_inches='tight')
1564
+
1565
+ plt.show()
1566
+
1567
+ # %% [code]
1568
+ if not test_only:
1569
+ snapshot_download(repo_id=repo_name, local_dir="", repo_type="model", allow_patterns=["**/*lr*.json"], ignore_patterns=[])
1570
+ get_ipython().system('rm -rf .cache .gitattributes')
1571
+
1572
+ # %% [code]
1573
+ if not test_only:
1574
+ experiments = {}
1575
+ for experiment in os.listdir(pretrained_dir):
1576
+ experiment_logs = []
1577
+ try:
1578
+ for seed in SEEDS:
1579
+ for fold_idx in range(nfolds):
1580
+ with open(f"{pretrained_dir}/{experiment}/logs/{experiment}_s{seed}_f{fold_idx}_logging.json", "r", encoding="utf-8") as f:
1581
+ experiment_log = json.load(f)
1582
+ experiment_logs.append(experiment_log)
1583
+ except:
1584
+ pass
1585
+ experiments[experiment] = get_avg_log(experiment_logs, 1000)
1586
+ experiments[state_dict_save_name] = get_avg_log(training_logs, 1000)
1587
+
1588
+ # %% [code]
1589
+ if not test_only:
1590
+ score = get_avg_best_score(training_logs)
1591
+ state_dict_save_name, score
1592
+
1593
+ # %% [code]
1594
+ if not test_only:
1595
+ plot_training_logs(experiments, save_path=f'{checkpoints_dir}/logs/{state_dict_save_name}_log_plot.jpg', figsize=(18, 7.5))
1596
+
11_12_13_lr_17/lasts/11_12_13_lr_17_s26092004_f0_last_ema.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b8a3409d264f7a4222de57db4545605eab9c8185484a5ed4b35bf035b8f6a598
3
+ size 544007234
11_12_13_lr_17/logs/11_12_13_lr_17_log_plot.jpg ADDED

Git LFS Details

  • SHA256: 8f14ebf9849ab003beffa6cbf85207b6a4552ebd084962f848d0ce058e9c6c35
  • Pointer size: 131 Bytes
  • Size of remote file: 580 kB
11_12_13_lr_17/logs/11_12_13_lr_17_s26092004_f0_logging.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"1": {"lr": [2e-05, 0.0005], "train_loss": 3.849170446395874, "total": 3.8491704934416804, "contrastive_loss": 3.708609182300376, "triplet_loss": 0.2729933110367893}, "2": {"lr": [1.988303923565381e-05, 0.0004969282409784868], "train_loss": 3.630740165710449, "total": 3.630740162520903, "contrastive_loss": 3.49398221937709, "triplet_loss": 0.266304347826087}, "3": {"lr": [1.9535036904803962e-05, 0.0004877886008156408], "train_loss": 3.451627492904663, "total": 3.451627495296823, "contrastive_loss": 3.2980424249451294, "triplet_loss": 0.30351170568561875}, "4": {"lr": [1.8964561979789496e-05, 0.00047280612778499774], "train_loss": 3.2030296325683594, "total": 3.203029670842914, "contrastive_loss": 3.057277309455999, "triplet_loss": 0.28658026755852845}, "5": {"lr": [1.8185661446562005e-05, 0.00045234974009654937], "train_loss": 3.1770193576812744, "total": 3.1770193042563752, "contrastive_loss": 3.030945844873537, "triplet_loss": 0.28532608695652173}, "6": {"lr": [1.7217514421272206e-05, 0.00042692314190604356], "train_loss": 2.969609022140503, "total": 2.9696089830685617, "contrastive_loss": 2.832015940178198, "triplet_loss": 0.27048494983277593}, "7": {"lr": [1.60839598967785e-05, 0.00039715242044697206], "train_loss": 2.9424145221710205, "total": 2.9424144527983906, "contrastive_loss": 2.7992476141173706, "triplet_loss": 0.2807274247491639}, "8": {"lr": [1.4812909747525698e-05, 0.00036377062968501693], "train_loss": 2.685025930404663, "total": 2.685025932796823, "contrastive_loss": 2.5528893103966346, "triplet_loss": 0.2614966555183946, "val_score": 0.831220238095238, "best_score": 0.831220238095238, "new_best_model": true, "recall": 0.831220238095238, "mAP": 0.3190260967813051, "mRP": 0.30719841269841264}, "9": {"lr": [1.3435661446562005e-05, 0.0003275997400965494], "train_loss": 2.5245449542999268, "total": 2.5245449104436664, "contrastive_loss": 2.3980667981814383, "triplet_loss": 0.2516722408026756, "val_score": 0.8322619047619048, "best_score": 0.8322619047619048, "new_best_model": true, "recall": 0.8322619047619048, "mAP": 0.32007002787225985, "mRP": 0.308483134920635}, "10": {"lr": [1.1986127417882198e-05, 0.00028953039902753766], "train_loss": 2.3836095333099365, "total": 2.3836096313884823, "contrastive_loss": 2.2628249356579224, "triplet_loss": 0.2422658862876254, "val_score": 0.8289880952380952, "best_score": 0.8322619047619048, "new_best_model": false, "recall": 0.8289880952380952, "mAP": 0.3180637944066515, "mRP": 0.30396230158730164}, "11": {"lr": [1.0500000000000003e-05, 0.0002505], "train_loss": 2.261930465698242, "total": 2.26193043380278, "contrastive_loss": 2.1466256336224916, "triplet_loss": 0.23244147157190637, "val_score": 0.8221130952380953, "best_score": 0.8322619047619048, "new_best_model": false, "recall": 0.8221130952380953, "mAP": 0.3159932732583775, "mRP": 0.30088839285714297}, "12": {"lr": [9.013872582117811e-06, 0.00021146960097246246], "train_loss": 2.1652767658233643, "total": 2.1652766709343645, "contrastive_loss": 2.0542114666074416, "triplet_loss": 0.22240802675585283, "val_score": 0.8117559523809524, "best_score": 0.8322619047619048, "new_best_model": false, "recall": 0.8117559523809524, "mAP": 0.31422275604686317, "mRP": 0.29918204365079365}, "13": {"lr": [7.564338553438001e-06, 0.00017340025990345064], "train_loss": 2.090695858001709, "total": 2.0906957926160117, "contrastive_loss": 1.9831438861961748, "triplet_loss": 0.21509197324414717, "val_score": 0.802797619047619, "best_score": 0.8322619047619048, "new_best_model": false, "recall": 0.802797619047619, "mAP": 0.3094924176036157, "mRP": 0.2945724206349206}, "14": {"lr": [6.1870902524743065e-06, 0.00013722937031498307], "train_loss": 2.03192138671875, "total": 2.03192138671875, "contrastive_loss": 1.9271781181411998, "triplet_loss": 0.20903010033444816, "val_score": 0.792172619047619, "best_score": 0.8322619047619048, "new_best_model": false, "recall": 0.792172619047619, "mAP": 0.30488193342151665, "mRP": 0.2901661706349205}, "15": {"lr": [4.916040103221507e-06, 0.00010384757955302797], "train_loss": 1.9957189559936523, "total": 1.995718965562291, "contrastive_loss": 1.892721220801108, "triplet_loss": 0.205685618729097, "val_score": 0.7803571428571429, "best_score": 0.8322619047619048, "new_best_model": false, "recall": 0.7803571428571429, "mAP": 0.3015340746252204, "mRP": 0.28796626984126966}, "16": {"lr": [3.7824855787278e-06, 7.40768580939564e-05], "train_loss": 1.9728339910507202, "total": 1.9728340480638586, "contrastive_loss": 1.8708736968279682, "triplet_loss": 0.20317725752508362, "val_score": 0.7699404761904762, "best_score": 0.8322619047619048, "new_best_model": false, "recall": 0.7699404761904762, "mAP": 0.294843858851411, "mRP": 0.2823516865079364}, "17": {"lr": [2.814338553438001e-06, 4.865025990345063e-05], "train_loss": 1.9708184003829956, "total": 1.9708184589909072, "contrastive_loss": 1.868879337374582, "triplet_loss": 0.20286371237458195, "val_score": 0.7593154761904762, "best_score": 0.8322619047619048, "new_best_model": false, "recall": 0.7593154761904762, "mAP": 0.2890880621693121, "mRP": 0.2758794642857142}, "18": {"lr": [2.0354380202105066e-06, 2.8193872215002235e-05], "train_loss": 1.9804123640060425, "total": 1.9804124098557692, "contrastive_loss": 1.8776865675297867, "triplet_loss": 0.2044314381270903, "val_score": 0.7507738095238096, "best_score": 0.8322619047619048, "new_best_model": false, "recall": 0.7507738095238096, "mAP": 0.2841381324404762, "mRP": 0.2717440476190474}}
11_12_13_lr_17/r1s/11_12_13_lr_17_s26092004_f0_r1_vs0.83226_ema.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2657f0460363008bf513b03618d6c3ca27d468b2e20f2e3663a0877a61c87ab0
3
+ size 544008938
11_12_13_lr_17/results/11_12_13_lr_17_test.json ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "Best model": {
3
+ "0": {
4
+ "5": {
5
+ "recall": 0.7153614457831325,
6
+ "mAP": 0.2913411981258386,
7
+ "mRP": 0.2956262550200819
8
+ },
9
+ "10": {
10
+ "recall": 0.7974397590361446,
11
+ "mAP": 0.3049957991688992,
12
+ "mRP": 0.2956262550200819
13
+ },
14
+ "15": {
15
+ "recall": 0.8414909638554217,
16
+ "mAP": 0.31045368256729466,
17
+ "mRP": 0.2956262550200819
18
+ }
19
+ },
20
+ "0.25": {
21
+ "5": {
22
+ "recall": 0.791039156626506,
23
+ "mAP": 0.32613955823293356,
24
+ "mRP": 0.3323920682730946
25
+ },
26
+ "10": {
27
+ "recall": 0.8678463855421686,
28
+ "mAP": 0.3421136206532493,
29
+ "mRP": 0.3323920682730946
30
+ },
31
+ "15": {
32
+ "recall": 0.8994728915662651,
33
+ "mAP": 0.34880642134736745,
34
+ "mRP": 0.3323920682730946
35
+ }
36
+ },
37
+ "0.5": {
38
+ "5": {
39
+ "recall": 0.7774849397590361,
40
+ "mAP": 0.31480296184739154,
41
+ "mRP": 0.3236132028112469
42
+ },
43
+ "10": {
44
+ "recall": 0.8550451807228916,
45
+ "mAP": 0.330907875051796,
46
+ "mRP": 0.3236132028112469
47
+ },
48
+ "15": {
49
+ "recall": 0.8926957831325302,
50
+ "mAP": 0.33725236546349235,
51
+ "mRP": 0.3236132028112469
52
+ }
53
+ },
54
+ "0.75": {
55
+ "5": {
56
+ "recall": 0.766566265060241,
57
+ "mAP": 0.3064808191097741,
58
+ "mRP": 0.312958082329319
59
+ },
60
+ "10": {
61
+ "recall": 0.8373493975903614,
62
+ "mAP": 0.3210760900745854,
63
+ "mRP": 0.312958082329319
64
+ },
65
+ "15": {
66
+ "recall": 0.875,
67
+ "mAP": 0.32729881814972295,
68
+ "mRP": 0.312958082329319
69
+ }
70
+ },
71
+ "1": {
72
+ "5": {
73
+ "recall": 0.7530120481927711,
74
+ "mAP": 0.2988659847724244,
75
+ "mRP": 0.3075112951807245
76
+ },
77
+ "10": {
78
+ "recall": 0.8283132530120482,
79
+ "mAP": 0.3133851004215285,
80
+ "mRP": 0.3075112951807245
81
+ },
82
+ "15": {
83
+ "recall": 0.8648343373493976,
84
+ "mAP": 0.3194783451537038,
85
+ "mRP": 0.3075112951807245
86
+ }
87
+ }
88
+ },
89
+ "Last model": {
90
+ "0": {
91
+ "5": {
92
+ "recall": 0.639683734939759,
93
+ "mAP": 0.2565855087014733,
94
+ "mRP": 0.2548820281124502
95
+ },
96
+ "10": {
97
+ "recall": 0.7206325301204819,
98
+ "mAP": 0.268452532849653,
99
+ "mRP": 0.2548820281124502
100
+ },
101
+ "15": {
102
+ "recall": 0.7594126506024096,
103
+ "mAP": 0.2727809764316734,
104
+ "mRP": 0.2548820281124502
105
+ }
106
+ },
107
+ "0.25": {
108
+ "5": {
109
+ "recall": 0.7816265060240963,
110
+ "mAP": 0.322587014725571,
111
+ "mRP": 0.3208270582329337
112
+ },
113
+ "10": {
114
+ "recall": 0.8595632530120482,
115
+ "mAP": 0.33856869691065966,
116
+ "mRP": 0.3208270582329337
117
+ },
118
+ "15": {
119
+ "recall": 0.8904367469879518,
120
+ "mAP": 0.3444647553368383,
121
+ "mRP": 0.3208270582329337
122
+ }
123
+ },
124
+ "0.5": {
125
+ "5": {
126
+ "recall": 0.7786144578313253,
127
+ "mAP": 0.3189318733266419,
128
+ "mRP": 0.3203376004016084
129
+ },
130
+ "10": {
131
+ "recall": 0.8509036144578314,
132
+ "mAP": 0.33440555127255184,
133
+ "mRP": 0.3203376004016084
134
+ },
135
+ "15": {
136
+ "recall": 0.8874246987951807,
137
+ "mAP": 0.34054143680602733,
138
+ "mRP": 0.3203376004016084
139
+ }
140
+ },
141
+ "0.75": {
142
+ "5": {
143
+ "recall": 0.7740963855421686,
144
+ "mAP": 0.31070375251004184,
145
+ "mRP": 0.3159073795180744
146
+ },
147
+ "10": {
148
+ "recall": 0.8418674698795181,
149
+ "mAP": 0.3254002816822859,
150
+ "mRP": 0.3159073795180744
151
+ },
152
+ "15": {
153
+ "recall": 0.8731174698795181,
154
+ "mAP": 0.33087249207731234,
155
+ "mRP": 0.3159073795180744
156
+ }
157
+ },
158
+ "1": {
159
+ "5": {
160
+ "recall": 0.7609186746987951,
161
+ "mAP": 0.3026845925368155,
162
+ "mRP": 0.30811997991968065
163
+ },
164
+ "10": {
165
+ "recall": 0.8320783132530121,
166
+ "mAP": 0.31732809711066584,
167
+ "mRP": 0.30811997991968065
168
+ },
169
+ "15": {
170
+ "recall": 0.8640813253012049,
171
+ "mAP": 0.32293537804927214,
172
+ "mRP": 0.30811997991968065
173
+ }
174
+ }
175
+ }
176
+ }
11_12_13_lr_17/results/11_12_13_lr_17_test_df.xlsx ADDED
Binary file (6.26 kB). View file
 
11_12_13_lr_17/results/11_12_13_lr_17_test_df_best.xlsx ADDED
Binary file (5.56 kB). View file