| { | |
| "model_config": { | |
| "model_name_or_path": "huawei-noah/TinyBERT_General_4L_312D", | |
| "attn_implementation": "flash_attention_2", | |
| "pooling": "mean", | |
| "head_type": "none", | |
| "normalize": true | |
| }, | |
| "loss_dict": { | |
| "name": "supcon", | |
| "loss_config": { | |
| "temperature": 0.01 | |
| }, | |
| "loss_optim_config": {} | |
| }, | |
| "lr": 0.0004, | |
| "head_lr_multiplier": 5.0, | |
| "weight_decay": 0.01, | |
| "beta1": 0.9, | |
| "beta2": 0.999, | |
| "eps": 1e-08, | |
| "lr_schedule": "linear", | |
| "warmup_ratio": 0.1, | |
| "compile": true, | |
| "microbatch_size": 256, | |
| "cross_batch_memory_size": null | |
| } | |