blackmlengineer commited on
Commit
57a5e46
·
verified ·
1 Parent(s): c924e9b

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. 1_Pooling/config.json +5 -0
  2. 2_Normalize/config.json +4 -0
  3. README.md +438 -0
  4. checkpoint-2600/1_Pooling/config.json +5 -0
  5. checkpoint-2600/2_Normalize/config.json +4 -0
  6. checkpoint-2600/README.md +433 -0
  7. checkpoint-2600/config.json +30 -0
  8. checkpoint-2600/config_sentence_transformers.json +14 -0
  9. checkpoint-2600/model.safetensors +3 -0
  10. checkpoint-2600/modules.json +20 -0
  11. checkpoint-2600/optimizer.pt +3 -0
  12. checkpoint-2600/rng_state.pth +3 -0
  13. checkpoint-2600/scheduler.pt +3 -0
  14. checkpoint-2600/sentence_bert_config.json +10 -0
  15. checkpoint-2600/tokenizer.json +0 -0
  16. checkpoint-2600/tokenizer_config.json +24 -0
  17. checkpoint-2600/trainer_state.json +286 -0
  18. checkpoint-2600/training_args.bin +3 -0
  19. checkpoint-3000/1_Pooling/config.json +5 -0
  20. checkpoint-3000/2_Normalize/config.json +4 -0
  21. checkpoint-3000/README.md +437 -0
  22. checkpoint-3000/config.json +30 -0
  23. checkpoint-3000/config_sentence_transformers.json +14 -0
  24. checkpoint-3000/model.safetensors +3 -0
  25. checkpoint-3000/modules.json +20 -0
  26. checkpoint-3000/optimizer.pt +3 -0
  27. checkpoint-3000/rng_state.pth +3 -0
  28. checkpoint-3000/scheduler.pt +3 -0
  29. checkpoint-3000/sentence_bert_config.json +10 -0
  30. checkpoint-3000/tokenizer.json +0 -0
  31. checkpoint-3000/tokenizer_config.json +24 -0
  32. checkpoint-3000/trainer_state.json +325 -0
  33. checkpoint-3000/training_args.bin +3 -0
  34. checkpoint-3100/1_Pooling/config.json +5 -0
  35. checkpoint-3100/2_Normalize/config.json +4 -0
  36. checkpoint-3100/README.md +438 -0
  37. checkpoint-3100/config.json +30 -0
  38. checkpoint-3100/config_sentence_transformers.json +14 -0
  39. checkpoint-3100/model.safetensors +3 -0
  40. checkpoint-3100/modules.json +20 -0
  41. checkpoint-3100/optimizer.pt +3 -0
  42. checkpoint-3100/rng_state.pth +3 -0
  43. checkpoint-3100/scheduler.pt +3 -0
  44. checkpoint-3100/sentence_bert_config.json +10 -0
  45. checkpoint-3100/tokenizer.json +0 -0
  46. checkpoint-3100/tokenizer_config.json +24 -0
  47. checkpoint-3100/trainer_state.json +333 -0
  48. checkpoint-3100/training_args.bin +3 -0
  49. config.json +30 -0
  50. config_sentence_transformers.json +14 -0
1_Pooling/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "embedding_dimension": 384,
3
+ "pooling_mode": "mean",
4
+ "include_prompt": true
5
+ }
2_Normalize/config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "module_input_name": "sentence_embedding",
3
+ "module_output_name": "sentence_embedding"
4
+ }
README.md ADDED
@@ -0,0 +1,438 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - en
4
+ tags:
5
+ - sentence-transformers
6
+ - sentence-similarity
7
+ - feature-extraction
8
+ - dense
9
+ - generated_from_trainer
10
+ - dataset_size:110608
11
+ - loss:MultipleNegativesRankingLoss
12
+ base_model: sentence-transformers/all-MiniLM-L6-v2
13
+ widget:
14
+ - source_sentence: proficiency in Yiddish required for this position
15
+ sentences:
16
+ - write Telugu
17
+ - Latvian
18
+ - Yiddish
19
+ - source_sentence: knowledge of international cinema is a necessity
20
+ sentences:
21
+ - SPARK
22
+ - Italian
23
+ - film studies
24
+ - source_sentence: experience with casting anode pins is a plus.
25
+ sentences:
26
+ - 'plan evaluation '
27
+ - synthetic natural environment
28
+ - cast cell covers
29
+ - source_sentence: Native speakers of Telugu are especially encouraged to apply.
30
+ sentences:
31
+ - Romanian
32
+ - Maltego
33
+ - Telugu
34
+ - source_sentence: looking for experienced animal hunters to join our team
35
+ sentences:
36
+ - animal hunting
37
+ - use databases
38
+ - electrical engineering
39
+ datasets:
40
+ - TechWolf/Synthetic-ESCO-skill-sentences
41
+ pipeline_tag: sentence-similarity
42
+ library_name: sentence-transformers
43
+ ---
44
+
45
+ # SentenceTransformer based on sentence-transformers/all-MiniLM-L6-v2
46
+
47
+ This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) on the [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) dataset. It maps inputs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, classification, clustering, and more.
48
+
49
+ ## Model Details
50
+
51
+ ### Model Description
52
+ - **Model Type:** Sentence Transformer
53
+ - **Base model:** [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) <!-- at revision 1110a243fdf4706b3f48f1d95db1a4f5529b4d41 -->
54
+ - **Maximum Sequence Length:** 256 tokens
55
+ - **Output Dimensionality:** 384 dimensions
56
+ - **Similarity Function:** Cosine Similarity
57
+ - **Supported Modality:** Text
58
+ - **Training Dataset:**
59
+ - [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences)
60
+ - **Language:** en
61
+ <!-- - **License:** Unknown -->
62
+
63
+ ### Model Sources
64
+
65
+ - **Documentation:** [Sentence Transformers Documentation](https://sbert.net)
66
+ - **Repository:** [Sentence Transformers on GitHub](https://github.com/huggingface/sentence-transformers)
67
+ - **Hugging Face:** [Sentence Transformers on Hugging Face](https://huggingface.co/models?library=sentence-transformers)
68
+
69
+ ### Full Model Architecture
70
+
71
+ ```
72
+ SentenceTransformer(
73
+ (0): Transformer({'transformer_task': 'feature-extraction', 'modality_config': {'text': {'method': 'forward', 'method_output_name': 'last_hidden_state'}}, 'module_output_name': 'token_embeddings', 'architecture': 'BertModel'})
74
+ (1): Pooling({'embedding_dimension': 384, 'pooling_mode': 'mean', 'include_prompt': True})
75
+ (2): Normalize({'module_input_name': 'sentence_embedding', 'module_output_name': 'sentence_embedding'})
76
+ )
77
+ ```
78
+
79
+ ## Usage
80
+
81
+ ### Direct Usage (Sentence Transformers)
82
+
83
+ First install the Sentence Transformers library:
84
+
85
+ ```bash
86
+ pip install -U sentence-transformers
87
+ ```
88
+ Then you can load this model and run inference.
89
+ ```python
90
+ from sentence_transformers import SentenceTransformer
91
+
92
+ # Download from the 🤗 Hub
93
+ model = SentenceTransformer("sentence_transformers_model_id")
94
+ # Run inference
95
+ sentences = [
96
+ 'looking for experienced animal hunters to join our team',
97
+ 'animal hunting',
98
+ 'electrical engineering',
99
+ ]
100
+ embeddings = model.encode(sentences)
101
+ print(embeddings.shape)
102
+ # [3, 384]
103
+
104
+ # Get the similarity scores for the embeddings
105
+ similarities = model.similarity(embeddings, embeddings)
106
+ print(similarities)
107
+ # tensor([[ 1.0000, 0.8278, -0.1120],
108
+ # [ 0.8278, 1.0000, -0.1666],
109
+ # [-0.1120, -0.1666, 1.0000]])
110
+ ```
111
+ <!--
112
+ ### Direct Usage (Transformers)
113
+
114
+ <details><summary>Click to see the direct usage in Transformers</summary>
115
+
116
+ </details>
117
+ -->
118
+
119
+ <!--
120
+ ### Downstream Usage (Sentence Transformers)
121
+
122
+ You can finetune this model on your own dataset.
123
+
124
+ <details><summary>Click to expand</summary>
125
+
126
+ </details>
127
+ -->
128
+
129
+ <!--
130
+ ### Out-of-Scope Use
131
+
132
+ *List how the model may foreseeably be misused and address what users ought not to do with the model.*
133
+ -->
134
+
135
+ <!--
136
+ ## Bias, Risks and Limitations
137
+
138
+ *What are the known or foreseeable issues stemming from this model? You could also flag here known failure cases or weaknesses of the model.*
139
+ -->
140
+
141
+ <!--
142
+ ### Recommendations
143
+
144
+ *What are recommendations with respect to the foreseeable issues? For example, filtering explicit content.*
145
+ -->
146
+
147
+ ## Training Details
148
+
149
+ ### Training Dataset
150
+
151
+ #### synthetic-esco-skill-sentences
152
+
153
+ * Dataset: [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) at [09fc487](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences/tree/09fc487d1394c1fdfd224cb621a89b14af925716)
154
+ * Size: 110,608 training samples
155
+ * Columns: <code>anchor</code> and <code>positive</code>
156
+ * Approximate statistics based on the first 100 samples:
157
+ | | anchor | positive |
158
+ |:---------|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|
159
+ | type | string | string |
160
+ | modality | text | text |
161
+ | details | <ul><li>min: 11 tokens</li><li>mean: 19.69 tokens</li><li>max: 30 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 6.53 tokens</li><li>max: 15 tokens</li></ul> |
162
+ * Samples:
163
+ | anchor | positive |
164
+ |:-----------------------------------------------------------------------------------------------------------------|:--------------------------------------------|
165
+ | <code>Applicants should be skilled in examining legislative drafts for legal compliance and accuracy.</code> | <code>examine legislative drafts</code> |
166
+ | <code>Candidate must have knowledge in photographic film processing, including monitoring chemical baths.</code> | <code>monitor film development baths</code> |
167
+ | <code>we are looking for an experienced meat grinder to join our team</code> | <code>grind meat</code> |
168
+ * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
169
+ ```json
170
+ {
171
+ "scale": 20.0,
172
+ "similarity_fct": "cos_sim",
173
+ "gather_across_devices": false,
174
+ "directions": [
175
+ "query_to_doc"
176
+ ],
177
+ "partition_mode": "joint",
178
+ "hardness_mode": null,
179
+ "hardness_strength": 0.0
180
+ }
181
+ ```
182
+
183
+ ### Evaluation Dataset
184
+
185
+ #### synthetic-esco-skill-sentences
186
+
187
+ * Dataset: [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) at [09fc487](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences/tree/09fc487d1394c1fdfd224cb621a89b14af925716)
188
+ * Size: 27,652 evaluation samples
189
+ * Columns: <code>anchor</code> and <code>positive</code>
190
+ * Approximate statistics based on the first 100 samples:
191
+ | | anchor | positive |
192
+ |:---------|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|
193
+ | type | string | string |
194
+ | modality | text | text |
195
+ | details | <ul><li>min: 11 tokens</li><li>mean: 19.55 tokens</li><li>max: 34 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 5.96 tokens</li><li>max: 10 tokens</li></ul> |
196
+ * Samples:
197
+ | anchor | positive |
198
+ |:-----------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------|
199
+ | <code>the position requires attention to detail and strong organizational skills in order to maintain accurate records of security incidents.</code> | <code>document security incidents in the store</code> |
200
+ | <code>We need someone with experience in export logistics to lead our transportation team</code> | <code>coordinate export transportation activities</code> |
201
+ | <code>The ideal candidate will have experience in developing food policy that meets the social objectives of our company.</code> | <code>develop food policy</code> |
202
+ * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
203
+ ```json
204
+ {
205
+ "scale": 20.0,
206
+ "similarity_fct": "cos_sim",
207
+ "gather_across_devices": false,
208
+ "directions": [
209
+ "query_to_doc"
210
+ ],
211
+ "partition_mode": "joint",
212
+ "hardness_mode": null,
213
+ "hardness_strength": 0.0
214
+ }
215
+ ```
216
+
217
+ ### Training Hyperparameters
218
+ #### Non-Default Hyperparameters
219
+
220
+ - `per_device_train_batch_size`: 128
221
+ - `num_train_epochs`: 50
222
+ - `learning_rate`: 0.0001
223
+ - `weight_decay`: 0.01
224
+ - `per_device_eval_batch_size`: 128
225
+ - `dataloader_pin_memory`: False
226
+
227
+ #### All Hyperparameters
228
+ <details><summary>Click to expand</summary>
229
+
230
+ - `per_device_train_batch_size`: 128
231
+ - `num_train_epochs`: 50
232
+ - `max_steps`: -1
233
+ - `learning_rate`: 0.0001
234
+ - `lr_scheduler_type`: linear
235
+ - `lr_scheduler_kwargs`: None
236
+ - `warmup_steps`: 0
237
+ - `optim`: adamw_torch_fused
238
+ - `optim_args`: None
239
+ - `weight_decay`: 0.01
240
+ - `adam_beta1`: 0.9
241
+ - `adam_beta2`: 0.999
242
+ - `adam_epsilon`: 1e-08
243
+ - `optim_target_modules`: None
244
+ - `gradient_accumulation_steps`: 1
245
+ - `average_tokens_across_devices`: True
246
+ - `max_grad_norm`: 1.0
247
+ - `label_smoothing_factor`: 0.0
248
+ - `bf16`: False
249
+ - `fp16`: False
250
+ - `bf16_full_eval`: False
251
+ - `fp16_full_eval`: False
252
+ - `tf32`: None
253
+ - `gradient_checkpointing`: False
254
+ - `gradient_checkpointing_kwargs`: None
255
+ - `torch_compile`: False
256
+ - `torch_compile_backend`: None
257
+ - `torch_compile_mode`: None
258
+ - `use_liger_kernel`: False
259
+ - `liger_kernel_config`: None
260
+ - `use_cache`: False
261
+ - `neftune_noise_alpha`: None
262
+ - `torch_empty_cache_steps`: None
263
+ - `auto_find_batch_size`: False
264
+ - `log_on_each_node`: True
265
+ - `logging_nan_inf_filter`: True
266
+ - `include_num_input_tokens_seen`: no
267
+ - `log_level`: passive
268
+ - `log_level_replica`: warning
269
+ - `disable_tqdm`: False
270
+ - `project`: huggingface
271
+ - `trackio_space_id`: None
272
+ - `trackio_bucket_id`: None
273
+ - `trackio_static_space_id`: None
274
+ - `per_device_eval_batch_size`: 128
275
+ - `prediction_loss_only`: True
276
+ - `eval_on_start`: False
277
+ - `eval_do_concat_batches`: True
278
+ - `eval_use_gather_object`: False
279
+ - `eval_accumulation_steps`: None
280
+ - `include_for_metrics`: []
281
+ - `batch_eval_metrics`: False
282
+ - `save_only_model`: False
283
+ - `save_on_each_node`: False
284
+ - `enable_jit_checkpoint`: False
285
+ - `push_to_hub`: False
286
+ - `hub_private_repo`: None
287
+ - `hub_model_id`: None
288
+ - `hub_strategy`: every_save
289
+ - `hub_always_push`: False
290
+ - `hub_revision`: None
291
+ - `load_best_model_at_end`: False
292
+ - `ignore_data_skip`: False
293
+ - `restore_callback_states_from_checkpoint`: False
294
+ - `full_determinism`: False
295
+ - `seed`: 42
296
+ - `data_seed`: None
297
+ - `use_cpu`: False
298
+ - `accelerator_config`: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
299
+ - `parallelism_config`: None
300
+ - `dataloader_drop_last`: False
301
+ - `dataloader_num_workers`: 0
302
+ - `dataloader_pin_memory`: False
303
+ - `dataloader_persistent_workers`: False
304
+ - `dataloader_prefetch_factor`: None
305
+ - `dataloader_multiprocessing_context`: None
306
+ - `dataloader_in_order`: True
307
+ - `remove_unused_columns`: True
308
+ - `label_names`: None
309
+ - `train_sampling_strategy`: random
310
+ - `length_column_name`: length
311
+ - `ddp_find_unused_parameters`: None
312
+ - `ddp_bucket_cap_mb`: None
313
+ - `ddp_broadcast_buffers`: False
314
+ - `ddp_static_graph`: None
315
+ - `ddp_backend`: None
316
+ - `ddp_timeout`: 1800
317
+ - `fsdp`: None
318
+ - `fsdp_config`: None
319
+ - `deepspeed`: None
320
+ - `debug`: []
321
+ - `skip_memory_metrics`: True
322
+ - `do_predict`: False
323
+ - `resume_from_checkpoint`: None
324
+ - `local_rank`: -1
325
+ - `prompts`: None
326
+ - `batch_sampler`: batch_sampler
327
+ - `multi_dataset_batch_sampler`: proportional
328
+ - `router_mapping`: {}
329
+ - `learning_rate_mapping`: {}
330
+ - `warmup_ratio`: None
331
+
332
+ </details>
333
+
334
+ ### Training Logs
335
+ | Epoch | Step | Training Loss | Validation Loss |
336
+ |:------:|:----:|:-------------:|:---------------:|
337
+ | 0.1156 | 100 | - | 0.2757 |
338
+ | 0.2312 | 200 | - | 0.2636 |
339
+ | 0.3468 | 300 | - | 0.2548 |
340
+ | 0.4624 | 400 | - | 0.2484 |
341
+ | 0.5780 | 500 | 0.2929 | 0.2452 |
342
+ | 0.6936 | 600 | - | 0.2387 |
343
+ | 0.8092 | 700 | - | 0.2291 |
344
+ | 0.9249 | 800 | - | 0.2355 |
345
+ | 1.0405 | 900 | - | 0.2288 |
346
+ | 1.1561 | 1000 | 0.2378 | 0.2275 |
347
+ | 1.2717 | 1100 | - | 0.2263 |
348
+ | 1.3873 | 1200 | - | 0.2282 |
349
+ | 1.5029 | 1300 | - | 0.2228 |
350
+ | 1.6185 | 1400 | - | 0.2199 |
351
+ | 1.7341 | 1500 | 0.1797 | 0.2235 |
352
+ | 1.8497 | 1600 | - | 0.2181 |
353
+ | 1.9653 | 1700 | - | 0.2214 |
354
+ | 2.0809 | 1800 | - | 0.2164 |
355
+ | 2.1965 | 1900 | - | 0.2150 |
356
+ | 2.3121 | 2000 | 0.1466 | 0.2133 |
357
+ | 2.4277 | 2100 | - | 0.2122 |
358
+ | 2.5434 | 2200 | - | 0.2117 |
359
+ | 2.6590 | 2300 | - | 0.2118 |
360
+ | 2.7746 | 2400 | - | 0.2113 |
361
+ | 2.8902 | 2500 | 0.1352 | 0.2097 |
362
+ | 3.0058 | 2600 | - | 0.2075 |
363
+ | 3.1214 | 2700 | - | 0.2110 |
364
+ | 3.2370 | 2800 | - | 0.2114 |
365
+ | 3.3526 | 2900 | - | 0.2170 |
366
+ | 3.4682 | 3000 | 0.1012 | 0.2143 |
367
+ | 3.5838 | 3100 | - | 0.2110 |
368
+
369
+
370
+ ### Training Time
371
+ - **Training**: 20.9 minutes
372
+ - **Evaluation**: 13.3 minutes
373
+ - **Total**: 34.2 minutes
374
+
375
+ ### Framework Versions
376
+ - Python: 3.14.7
377
+ - Sentence Transformers: 6.2.0
378
+ - Transformers: 5.22.0
379
+ - PyTorch: 2.15.0
380
+ - Accelerate: 1.15.0
381
+ - Datasets: 5.1.0
382
+ - Tokenizers: 0.23.3
383
+
384
+ ## Additional Resources
385
+
386
+ - [Training and Finetuning Embedding Models with Sentence Transformers](https://huggingface.co/blog/train-sentence-transformers): the end-to-end guide for training or finetuning Sentence Transformer models.
387
+ - [Introduction to Matryoshka Embedding Models](https://huggingface.co/blog/matryoshka): variable-size embeddings that can be truncated with minimal quality loss.
388
+ - [Binary and Scalar Embedding Quantization for Significantly Faster & Cheaper Retrieval](https://huggingface.co/blog/embedding-quantization): post-training compression of embedding vectors.
389
+ - [Multimodal Embedding & Reranker Models with Sentence Transformers](https://huggingface.co/blog/multimodal-sentence-transformers): use text, image, audio, and video models through the same API.
390
+ - [Training and Finetuning Multimodal Embedding & Reranker Models with Sentence Transformers](https://huggingface.co/blog/train-multimodal-sentence-transformers): train multimodal embedding models, with a Visual Document Retrieval walkthrough.
391
+
392
+ ## Citation
393
+
394
+ ### BibTeX
395
+
396
+ #### Sentence Transformers
397
+ ```bibtex
398
+ @inproceedings{reimers-2019-sentence-bert,
399
+ title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
400
+ author = "Reimers, Nils and Gurevych, Iryna",
401
+ booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
402
+ month = "11",
403
+ year = "2019",
404
+ publisher = "Association for Computational Linguistics",
405
+ url = "https://arxiv.org/abs/1908.10084",
406
+ }
407
+ ```
408
+
409
+ #### MultipleNegativesRankingLoss
410
+ ```bibtex
411
+ @misc{oord2019representationlearningcontrastivepredictive,
412
+ title={Representation Learning with Contrastive Predictive Coding},
413
+ author={Aaron van den Oord and Yazhe Li and Oriol Vinyals},
414
+ year={2019},
415
+ eprint={1807.03748},
416
+ archivePrefix={arXiv},
417
+ primaryClass={cs.LG},
418
+ url={https://arxiv.org/abs/1807.03748},
419
+ }
420
+ ```
421
+
422
+ <!--
423
+ ## Glossary
424
+
425
+ *Clearly define terms in order to be accessible across audiences.*
426
+ -->
427
+
428
+ <!--
429
+ ## Model Card Authors
430
+
431
+ *Lists the people who create the model card, providing recognition and accountability for the detailed work that goes into its construction.*
432
+ -->
433
+
434
+ <!--
435
+ ## Model Card Contact
436
+
437
+ *Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors.*
438
+ -->
checkpoint-2600/1_Pooling/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "embedding_dimension": 384,
3
+ "pooling_mode": "mean",
4
+ "include_prompt": true
5
+ }
checkpoint-2600/2_Normalize/config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "module_input_name": "sentence_embedding",
3
+ "module_output_name": "sentence_embedding"
4
+ }
checkpoint-2600/README.md ADDED
@@ -0,0 +1,433 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - en
4
+ tags:
5
+ - sentence-transformers
6
+ - sentence-similarity
7
+ - feature-extraction
8
+ - dense
9
+ - generated_from_trainer
10
+ - dataset_size:110608
11
+ - loss:MultipleNegativesRankingLoss
12
+ base_model: sentence-transformers/all-MiniLM-L6-v2
13
+ widget:
14
+ - source_sentence: proficiency in Yiddish required for this position
15
+ sentences:
16
+ - write Telugu
17
+ - Latvian
18
+ - Yiddish
19
+ - source_sentence: knowledge of international cinema is a necessity
20
+ sentences:
21
+ - SPARK
22
+ - Italian
23
+ - film studies
24
+ - source_sentence: experience with casting anode pins is a plus.
25
+ sentences:
26
+ - 'plan evaluation '
27
+ - synthetic natural environment
28
+ - cast cell covers
29
+ - source_sentence: Native speakers of Telugu are especially encouraged to apply.
30
+ sentences:
31
+ - Romanian
32
+ - Maltego
33
+ - Telugu
34
+ - source_sentence: looking for experienced animal hunters to join our team
35
+ sentences:
36
+ - animal hunting
37
+ - use databases
38
+ - electrical engineering
39
+ datasets:
40
+ - TechWolf/Synthetic-ESCO-skill-sentences
41
+ pipeline_tag: sentence-similarity
42
+ library_name: sentence-transformers
43
+ ---
44
+
45
+ # SentenceTransformer based on sentence-transformers/all-MiniLM-L6-v2
46
+
47
+ This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) on the [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) dataset. It maps inputs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, classification, clustering, and more.
48
+
49
+ ## Model Details
50
+
51
+ ### Model Description
52
+ - **Model Type:** Sentence Transformer
53
+ - **Base model:** [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) <!-- at revision 1110a243fdf4706b3f48f1d95db1a4f5529b4d41 -->
54
+ - **Maximum Sequence Length:** 256 tokens
55
+ - **Output Dimensionality:** 384 dimensions
56
+ - **Similarity Function:** Cosine Similarity
57
+ - **Supported Modality:** Text
58
+ - **Training Dataset:**
59
+ - [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences)
60
+ - **Language:** en
61
+ <!-- - **License:** Unknown -->
62
+
63
+ ### Model Sources
64
+
65
+ - **Documentation:** [Sentence Transformers Documentation](https://sbert.net)
66
+ - **Repository:** [Sentence Transformers on GitHub](https://github.com/huggingface/sentence-transformers)
67
+ - **Hugging Face:** [Sentence Transformers on Hugging Face](https://huggingface.co/models?library=sentence-transformers)
68
+
69
+ ### Full Model Architecture
70
+
71
+ ```
72
+ SentenceTransformer(
73
+ (0): Transformer({'transformer_task': 'feature-extraction', 'modality_config': {'text': {'method': 'forward', 'method_output_name': 'last_hidden_state'}}, 'module_output_name': 'token_embeddings', 'architecture': 'BertModel'})
74
+ (1): Pooling({'embedding_dimension': 384, 'pooling_mode': 'mean', 'include_prompt': True})
75
+ (2): Normalize({'module_input_name': 'sentence_embedding', 'module_output_name': 'sentence_embedding'})
76
+ )
77
+ ```
78
+
79
+ ## Usage
80
+
81
+ ### Direct Usage (Sentence Transformers)
82
+
83
+ First install the Sentence Transformers library:
84
+
85
+ ```bash
86
+ pip install -U sentence-transformers
87
+ ```
88
+ Then you can load this model and run inference.
89
+ ```python
90
+ from sentence_transformers import SentenceTransformer
91
+
92
+ # Download from the 🤗 Hub
93
+ model = SentenceTransformer("sentence_transformers_model_id")
94
+ # Run inference
95
+ sentences = [
96
+ 'looking for experienced animal hunters to join our team',
97
+ 'animal hunting',
98
+ 'electrical engineering',
99
+ ]
100
+ embeddings = model.encode(sentences)
101
+ print(embeddings.shape)
102
+ # [3, 384]
103
+
104
+ # Get the similarity scores for the embeddings
105
+ similarities = model.similarity(embeddings, embeddings)
106
+ print(similarities)
107
+ # tensor([[ 1.0000, 0.8443, -0.1066],
108
+ # [ 0.8443, 1.0000, -0.1551],
109
+ # [-0.1066, -0.1551, 1.0000]])
110
+ ```
111
+ <!--
112
+ ### Direct Usage (Transformers)
113
+
114
+ <details><summary>Click to see the direct usage in Transformers</summary>
115
+
116
+ </details>
117
+ -->
118
+
119
+ <!--
120
+ ### Downstream Usage (Sentence Transformers)
121
+
122
+ You can finetune this model on your own dataset.
123
+
124
+ <details><summary>Click to expand</summary>
125
+
126
+ </details>
127
+ -->
128
+
129
+ <!--
130
+ ### Out-of-Scope Use
131
+
132
+ *List how the model may foreseeably be misused and address what users ought not to do with the model.*
133
+ -->
134
+
135
+ <!--
136
+ ## Bias, Risks and Limitations
137
+
138
+ *What are the known or foreseeable issues stemming from this model? You could also flag here known failure cases or weaknesses of the model.*
139
+ -->
140
+
141
+ <!--
142
+ ### Recommendations
143
+
144
+ *What are recommendations with respect to the foreseeable issues? For example, filtering explicit content.*
145
+ -->
146
+
147
+ ## Training Details
148
+
149
+ ### Training Dataset
150
+
151
+ #### synthetic-esco-skill-sentences
152
+
153
+ * Dataset: [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) at [09fc487](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences/tree/09fc487d1394c1fdfd224cb621a89b14af925716)
154
+ * Size: 110,608 training samples
155
+ * Columns: <code>anchor</code> and <code>positive</code>
156
+ * Approximate statistics based on the first 100 samples:
157
+ | | anchor | positive |
158
+ |:---------|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|
159
+ | type | string | string |
160
+ | modality | text | text |
161
+ | details | <ul><li>min: 11 tokens</li><li>mean: 19.69 tokens</li><li>max: 30 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 6.53 tokens</li><li>max: 15 tokens</li></ul> |
162
+ * Samples:
163
+ | anchor | positive |
164
+ |:-----------------------------------------------------------------------------------------------------------------|:--------------------------------------------|
165
+ | <code>Applicants should be skilled in examining legislative drafts for legal compliance and accuracy.</code> | <code>examine legislative drafts</code> |
166
+ | <code>Candidate must have knowledge in photographic film processing, including monitoring chemical baths.</code> | <code>monitor film development baths</code> |
167
+ | <code>we are looking for an experienced meat grinder to join our team</code> | <code>grind meat</code> |
168
+ * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
169
+ ```json
170
+ {
171
+ "scale": 20.0,
172
+ "similarity_fct": "cos_sim",
173
+ "gather_across_devices": false,
174
+ "directions": [
175
+ "query_to_doc"
176
+ ],
177
+ "partition_mode": "joint",
178
+ "hardness_mode": null,
179
+ "hardness_strength": 0.0
180
+ }
181
+ ```
182
+
183
+ ### Evaluation Dataset
184
+
185
+ #### synthetic-esco-skill-sentences
186
+
187
+ * Dataset: [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) at [09fc487](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences/tree/09fc487d1394c1fdfd224cb621a89b14af925716)
188
+ * Size: 27,652 evaluation samples
189
+ * Columns: <code>anchor</code> and <code>positive</code>
190
+ * Approximate statistics based on the first 100 samples:
191
+ | | anchor | positive |
192
+ |:---------|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|
193
+ | type | string | string |
194
+ | modality | text | text |
195
+ | details | <ul><li>min: 11 tokens</li><li>mean: 19.55 tokens</li><li>max: 34 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 5.96 tokens</li><li>max: 10 tokens</li></ul> |
196
+ * Samples:
197
+ | anchor | positive |
198
+ |:-----------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------|
199
+ | <code>the position requires attention to detail and strong organizational skills in order to maintain accurate records of security incidents.</code> | <code>document security incidents in the store</code> |
200
+ | <code>We need someone with experience in export logistics to lead our transportation team</code> | <code>coordinate export transportation activities</code> |
201
+ | <code>The ideal candidate will have experience in developing food policy that meets the social objectives of our company.</code> | <code>develop food policy</code> |
202
+ * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
203
+ ```json
204
+ {
205
+ "scale": 20.0,
206
+ "similarity_fct": "cos_sim",
207
+ "gather_across_devices": false,
208
+ "directions": [
209
+ "query_to_doc"
210
+ ],
211
+ "partition_mode": "joint",
212
+ "hardness_mode": null,
213
+ "hardness_strength": 0.0
214
+ }
215
+ ```
216
+
217
+ ### Training Hyperparameters
218
+ #### Non-Default Hyperparameters
219
+
220
+ - `per_device_train_batch_size`: 128
221
+ - `num_train_epochs`: 50
222
+ - `learning_rate`: 0.0001
223
+ - `weight_decay`: 0.01
224
+ - `per_device_eval_batch_size`: 128
225
+ - `dataloader_pin_memory`: False
226
+
227
+ #### All Hyperparameters
228
+ <details><summary>Click to expand</summary>
229
+
230
+ - `per_device_train_batch_size`: 128
231
+ - `num_train_epochs`: 50
232
+ - `max_steps`: -1
233
+ - `learning_rate`: 0.0001
234
+ - `lr_scheduler_type`: linear
235
+ - `lr_scheduler_kwargs`: None
236
+ - `warmup_steps`: 0
237
+ - `optim`: adamw_torch_fused
238
+ - `optim_args`: None
239
+ - `weight_decay`: 0.01
240
+ - `adam_beta1`: 0.9
241
+ - `adam_beta2`: 0.999
242
+ - `adam_epsilon`: 1e-08
243
+ - `optim_target_modules`: None
244
+ - `gradient_accumulation_steps`: 1
245
+ - `average_tokens_across_devices`: True
246
+ - `max_grad_norm`: 1.0
247
+ - `label_smoothing_factor`: 0.0
248
+ - `bf16`: False
249
+ - `fp16`: False
250
+ - `bf16_full_eval`: False
251
+ - `fp16_full_eval`: False
252
+ - `tf32`: None
253
+ - `gradient_checkpointing`: False
254
+ - `gradient_checkpointing_kwargs`: None
255
+ - `torch_compile`: False
256
+ - `torch_compile_backend`: None
257
+ - `torch_compile_mode`: None
258
+ - `use_liger_kernel`: False
259
+ - `liger_kernel_config`: None
260
+ - `use_cache`: False
261
+ - `neftune_noise_alpha`: None
262
+ - `torch_empty_cache_steps`: None
263
+ - `auto_find_batch_size`: False
264
+ - `log_on_each_node`: True
265
+ - `logging_nan_inf_filter`: True
266
+ - `include_num_input_tokens_seen`: no
267
+ - `log_level`: passive
268
+ - `log_level_replica`: warning
269
+ - `disable_tqdm`: False
270
+ - `project`: huggingface
271
+ - `trackio_space_id`: None
272
+ - `trackio_bucket_id`: None
273
+ - `trackio_static_space_id`: None
274
+ - `per_device_eval_batch_size`: 128
275
+ - `prediction_loss_only`: True
276
+ - `eval_on_start`: False
277
+ - `eval_do_concat_batches`: True
278
+ - `eval_use_gather_object`: False
279
+ - `eval_accumulation_steps`: None
280
+ - `include_for_metrics`: []
281
+ - `batch_eval_metrics`: False
282
+ - `save_only_model`: False
283
+ - `save_on_each_node`: False
284
+ - `enable_jit_checkpoint`: False
285
+ - `push_to_hub`: False
286
+ - `hub_private_repo`: None
287
+ - `hub_model_id`: None
288
+ - `hub_strategy`: every_save
289
+ - `hub_always_push`: False
290
+ - `hub_revision`: None
291
+ - `load_best_model_at_end`: False
292
+ - `ignore_data_skip`: False
293
+ - `restore_callback_states_from_checkpoint`: False
294
+ - `full_determinism`: False
295
+ - `seed`: 42
296
+ - `data_seed`: None
297
+ - `use_cpu`: False
298
+ - `accelerator_config`: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
299
+ - `parallelism_config`: None
300
+ - `dataloader_drop_last`: False
301
+ - `dataloader_num_workers`: 0
302
+ - `dataloader_pin_memory`: False
303
+ - `dataloader_persistent_workers`: False
304
+ - `dataloader_prefetch_factor`: None
305
+ - `dataloader_multiprocessing_context`: None
306
+ - `dataloader_in_order`: True
307
+ - `remove_unused_columns`: True
308
+ - `label_names`: None
309
+ - `train_sampling_strategy`: random
310
+ - `length_column_name`: length
311
+ - `ddp_find_unused_parameters`: None
312
+ - `ddp_bucket_cap_mb`: None
313
+ - `ddp_broadcast_buffers`: False
314
+ - `ddp_static_graph`: None
315
+ - `ddp_backend`: None
316
+ - `ddp_timeout`: 1800
317
+ - `fsdp`: None
318
+ - `fsdp_config`: None
319
+ - `deepspeed`: None
320
+ - `debug`: []
321
+ - `skip_memory_metrics`: True
322
+ - `do_predict`: False
323
+ - `resume_from_checkpoint`: None
324
+ - `local_rank`: -1
325
+ - `prompts`: None
326
+ - `batch_sampler`: batch_sampler
327
+ - `multi_dataset_batch_sampler`: proportional
328
+ - `router_mapping`: {}
329
+ - `learning_rate_mapping`: {}
330
+ - `warmup_ratio`: None
331
+
332
+ </details>
333
+
334
+ ### Training Logs
335
+ | Epoch | Step | Training Loss | Validation Loss |
336
+ |:------:|:----:|:-------------:|:---------------:|
337
+ | 0.1156 | 100 | - | 0.2757 |
338
+ | 0.2312 | 200 | - | 0.2636 |
339
+ | 0.3468 | 300 | - | 0.2548 |
340
+ | 0.4624 | 400 | - | 0.2484 |
341
+ | 0.5780 | 500 | 0.2929 | 0.2452 |
342
+ | 0.6936 | 600 | - | 0.2387 |
343
+ | 0.8092 | 700 | - | 0.2291 |
344
+ | 0.9249 | 800 | - | 0.2355 |
345
+ | 1.0405 | 900 | - | 0.2288 |
346
+ | 1.1561 | 1000 | 0.2378 | 0.2275 |
347
+ | 1.2717 | 1100 | - | 0.2263 |
348
+ | 1.3873 | 1200 | - | 0.2282 |
349
+ | 1.5029 | 1300 | - | 0.2228 |
350
+ | 1.6185 | 1400 | - | 0.2199 |
351
+ | 1.7341 | 1500 | 0.1797 | 0.2235 |
352
+ | 1.8497 | 1600 | - | 0.2181 |
353
+ | 1.9653 | 1700 | - | 0.2214 |
354
+ | 2.0809 | 1800 | - | 0.2164 |
355
+ | 2.1965 | 1900 | - | 0.2150 |
356
+ | 2.3121 | 2000 | 0.1466 | 0.2133 |
357
+ | 2.4277 | 2100 | - | 0.2122 |
358
+ | 2.5434 | 2200 | - | 0.2117 |
359
+ | 2.6590 | 2300 | - | 0.2118 |
360
+ | 2.7746 | 2400 | - | 0.2113 |
361
+ | 2.8902 | 2500 | 0.1352 | 0.2097 |
362
+ | 3.0058 | 2600 | - | 0.2075 |
363
+
364
+
365
+ ### Training Time
366
+ - **Training**: 17.1 minutes
367
+ - **Evaluation**: 10.7 minutes
368
+ - **Total**: 27.8 minutes
369
+
370
+ ### Framework Versions
371
+ - Python: 3.14.7
372
+ - Sentence Transformers: 6.2.0
373
+ - Transformers: 5.22.0
374
+ - PyTorch: 2.15.0
375
+ - Accelerate: 1.15.0
376
+ - Datasets: 5.1.0
377
+ - Tokenizers: 0.23.3
378
+
379
+ ## Additional Resources
380
+
381
+ - [Training and Finetuning Embedding Models with Sentence Transformers](https://huggingface.co/blog/train-sentence-transformers): the end-to-end guide for training or finetuning Sentence Transformer models.
382
+ - [Introduction to Matryoshka Embedding Models](https://huggingface.co/blog/matryoshka): variable-size embeddings that can be truncated with minimal quality loss.
383
+ - [Binary and Scalar Embedding Quantization for Significantly Faster & Cheaper Retrieval](https://huggingface.co/blog/embedding-quantization): post-training compression of embedding vectors.
384
+ - [Multimodal Embedding & Reranker Models with Sentence Transformers](https://huggingface.co/blog/multimodal-sentence-transformers): use text, image, audio, and video models through the same API.
385
+ - [Training and Finetuning Multimodal Embedding & Reranker Models with Sentence Transformers](https://huggingface.co/blog/train-multimodal-sentence-transformers): train multimodal embedding models, with a Visual Document Retrieval walkthrough.
386
+
387
+ ## Citation
388
+
389
+ ### BibTeX
390
+
391
+ #### Sentence Transformers
392
+ ```bibtex
393
+ @inproceedings{reimers-2019-sentence-bert,
394
+ title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
395
+ author = "Reimers, Nils and Gurevych, Iryna",
396
+ booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
397
+ month = "11",
398
+ year = "2019",
399
+ publisher = "Association for Computational Linguistics",
400
+ url = "https://arxiv.org/abs/1908.10084",
401
+ }
402
+ ```
403
+
404
+ #### MultipleNegativesRankingLoss
405
+ ```bibtex
406
+ @misc{oord2019representationlearningcontrastivepredictive,
407
+ title={Representation Learning with Contrastive Predictive Coding},
408
+ author={Aaron van den Oord and Yazhe Li and Oriol Vinyals},
409
+ year={2019},
410
+ eprint={1807.03748},
411
+ archivePrefix={arXiv},
412
+ primaryClass={cs.LG},
413
+ url={https://arxiv.org/abs/1807.03748},
414
+ }
415
+ ```
416
+
417
+ <!--
418
+ ## Glossary
419
+
420
+ *Clearly define terms in order to be accessible across audiences.*
421
+ -->
422
+
423
+ <!--
424
+ ## Model Card Authors
425
+
426
+ *Lists the people who create the model card, providing recognition and accountability for the detailed work that goes into its construction.*
427
+ -->
428
+
429
+ <!--
430
+ ## Model Card Contact
431
+
432
+ *Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors.*
433
+ -->
checkpoint-2600/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_cross_attention": false,
3
+ "architectures": [
4
+ "BertModel"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": null,
8
+ "classifier_dropout": null,
9
+ "dtype": "float32",
10
+ "eos_token_id": null,
11
+ "gradient_checkpointing": false,
12
+ "hidden_act": "gelu",
13
+ "hidden_dropout_prob": 0.1,
14
+ "hidden_size": 384,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 1536,
17
+ "is_decoder": false,
18
+ "layer_norm_eps": 1e-12,
19
+ "max_position_embeddings": 512,
20
+ "model_type": "bert",
21
+ "num_attention_heads": 12,
22
+ "num_hidden_layers": 6,
23
+ "pad_token_id": 0,
24
+ "position_embedding_type": "absolute",
25
+ "tie_word_embeddings": true,
26
+ "transformers_version": "5.22.0",
27
+ "type_vocab_size": 2,
28
+ "use_cache": false,
29
+ "vocab_size": 30522
30
+ }
checkpoint-2600/config_sentence_transformers.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "__version__": {
3
+ "pytorch": "2.15.0",
4
+ "sentence_transformers": "6.2.0",
5
+ "transformers": "5.22.0"
6
+ },
7
+ "default_prompt_name": null,
8
+ "model_type": "SentenceTransformer",
9
+ "prompts": {
10
+ "document": "",
11
+ "query": ""
12
+ },
13
+ "similarity_fn_name": "cosine"
14
+ }
checkpoint-2600/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8029641509ad77c6db4db516bbc4dcf07c6ed19394ac2425148937fbe0c430b6
3
+ size 90864192
checkpoint-2600/modules.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "idx": 0,
4
+ "name": "0",
5
+ "path": "",
6
+ "type": "sentence_transformers.base.modules.transformer.Transformer"
7
+ },
8
+ {
9
+ "idx": 1,
10
+ "name": "1",
11
+ "path": "1_Pooling",
12
+ "type": "sentence_transformers.sentence_transformer.modules.pooling.Pooling"
13
+ },
14
+ {
15
+ "idx": 2,
16
+ "name": "2",
17
+ "path": "2_Normalize",
18
+ "type": "sentence_transformers.base.modules.normalize.Normalize"
19
+ }
20
+ ]
checkpoint-2600/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8eb16e6c48a1242e57d4f800b74a15aaaf6ea4aa8f85c392aa2c4149828c5358
3
+ size 180605387
checkpoint-2600/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ff36c60fa4fe77c8083faedd0fbe9bea2e501545bf62efbb26d607fd78bf59f
3
+ size 14391
checkpoint-2600/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc684d77d610132391c374534696929f266a6828df791e0b338acfefb7415bb5
3
+ size 1465
checkpoint-2600/sentence_bert_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "transformer_task": "feature-extraction",
3
+ "modality_config": {
4
+ "text": {
5
+ "method": "forward",
6
+ "method_output_name": "last_hidden_state"
7
+ }
8
+ },
9
+ "module_output_name": "token_embeddings"
10
+ }
checkpoint-2600/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-2600/tokenizer_config.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "cls_token": "[CLS]",
4
+ "do_basic_tokenize": true,
5
+ "do_lower_case": true,
6
+ "is_local": false,
7
+ "local_files_only": false,
8
+ "mask_token": "[MASK]",
9
+ "max_length": 128,
10
+ "model_max_length": 256,
11
+ "never_split": null,
12
+ "pad_to_multiple_of": null,
13
+ "pad_token": "[PAD]",
14
+ "pad_token_type_id": 0,
15
+ "padding_side": "right",
16
+ "sep_token": "[SEP]",
17
+ "stride": 0,
18
+ "strip_accents": null,
19
+ "tokenize_chinese_chars": true,
20
+ "tokenizer_class": "BertTokenizer",
21
+ "truncation_side": "right",
22
+ "truncation_strategy": "longest_first",
23
+ "unk_token": "[UNK]"
24
+ }
checkpoint-2600/trainer_state.json ADDED
@@ -0,0 +1,286 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 2600,
3
+ "best_metric": 0.2075337916612625,
4
+ "best_model_checkpoint": "ft/sentence-transformers/all-MiniLM-L6-v2/checkpoint-2600",
5
+ "epoch": 3.005780346820809,
6
+ "eval_steps": 100,
7
+ "global_step": 2600,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.11560693641618497,
14
+ "eval_loss": 0.2756878137588501,
15
+ "eval_runtime": 19.195,
16
+ "eval_samples_per_second": 1440.58,
17
+ "eval_steps_per_second": 11.305,
18
+ "step": 100
19
+ },
20
+ {
21
+ "epoch": 0.23121387283236994,
22
+ "eval_loss": 0.2635912597179413,
23
+ "eval_runtime": 19.1881,
24
+ "eval_samples_per_second": 1441.105,
25
+ "eval_steps_per_second": 11.309,
26
+ "step": 200
27
+ },
28
+ {
29
+ "epoch": 0.3468208092485549,
30
+ "eval_loss": 0.25484976172447205,
31
+ "eval_runtime": 20.4586,
32
+ "eval_samples_per_second": 1351.608,
33
+ "eval_steps_per_second": 10.607,
34
+ "step": 300
35
+ },
36
+ {
37
+ "epoch": 0.4624277456647399,
38
+ "eval_loss": 0.24843913316726685,
39
+ "eval_runtime": 24.2538,
40
+ "eval_samples_per_second": 1140.109,
41
+ "eval_steps_per_second": 8.947,
42
+ "step": 400
43
+ },
44
+ {
45
+ "epoch": 0.5780346820809249,
46
+ "grad_norm": 3.031534194946289,
47
+ "learning_rate": 9.884624277456648e-05,
48
+ "loss": 0.2928614196777344,
49
+ "step": 500
50
+ },
51
+ {
52
+ "epoch": 0.5780346820809249,
53
+ "eval_loss": 0.24522048234939575,
54
+ "eval_runtime": 24.3615,
55
+ "eval_samples_per_second": 1135.067,
56
+ "eval_steps_per_second": 8.907,
57
+ "step": 500
58
+ },
59
+ {
60
+ "epoch": 0.6936416184971098,
61
+ "eval_loss": 0.23871970176696777,
62
+ "eval_runtime": 23.7379,
63
+ "eval_samples_per_second": 1164.886,
64
+ "eval_steps_per_second": 9.141,
65
+ "step": 600
66
+ },
67
+ {
68
+ "epoch": 0.8092485549132948,
69
+ "eval_loss": 0.22912940382957458,
70
+ "eval_runtime": 23.4904,
71
+ "eval_samples_per_second": 1177.162,
72
+ "eval_steps_per_second": 9.238,
73
+ "step": 700
74
+ },
75
+ {
76
+ "epoch": 0.9248554913294798,
77
+ "eval_loss": 0.23552891612052917,
78
+ "eval_runtime": 23.3878,
79
+ "eval_samples_per_second": 1182.328,
80
+ "eval_steps_per_second": 9.278,
81
+ "step": 800
82
+ },
83
+ {
84
+ "epoch": 1.0404624277456647,
85
+ "eval_loss": 0.22875414788722992,
86
+ "eval_runtime": 24.2858,
87
+ "eval_samples_per_second": 1138.608,
88
+ "eval_steps_per_second": 8.935,
89
+ "step": 900
90
+ },
91
+ {
92
+ "epoch": 1.1560693641618498,
93
+ "grad_norm": 2.5727860927581787,
94
+ "learning_rate": 9.769017341040463e-05,
95
+ "loss": 0.2378170166015625,
96
+ "step": 1000
97
+ },
98
+ {
99
+ "epoch": 1.1560693641618498,
100
+ "eval_loss": 0.22745409607887268,
101
+ "eval_runtime": 24.6958,
102
+ "eval_samples_per_second": 1119.705,
103
+ "eval_steps_per_second": 8.787,
104
+ "step": 1000
105
+ },
106
+ {
107
+ "epoch": 1.2716763005780347,
108
+ "eval_loss": 0.22631044685840607,
109
+ "eval_runtime": 25.1507,
110
+ "eval_samples_per_second": 1099.453,
111
+ "eval_steps_per_second": 8.628,
112
+ "step": 1100
113
+ },
114
+ {
115
+ "epoch": 1.3872832369942196,
116
+ "eval_loss": 0.22824475169181824,
117
+ "eval_runtime": 25.1355,
118
+ "eval_samples_per_second": 1100.116,
119
+ "eval_steps_per_second": 8.633,
120
+ "step": 1200
121
+ },
122
+ {
123
+ "epoch": 1.5028901734104045,
124
+ "eval_loss": 0.22276242077350616,
125
+ "eval_runtime": 25.2896,
126
+ "eval_samples_per_second": 1093.414,
127
+ "eval_steps_per_second": 8.581,
128
+ "step": 1300
129
+ },
130
+ {
131
+ "epoch": 1.6184971098265897,
132
+ "eval_loss": 0.21992968022823334,
133
+ "eval_runtime": 25.7239,
134
+ "eval_samples_per_second": 1074.952,
135
+ "eval_steps_per_second": 8.436,
136
+ "step": 1400
137
+ },
138
+ {
139
+ "epoch": 1.7341040462427746,
140
+ "grad_norm": 2.2508647441864014,
141
+ "learning_rate": 9.653410404624278e-05,
142
+ "loss": 0.1796602783203125,
143
+ "step": 1500
144
+ },
145
+ {
146
+ "epoch": 1.7341040462427746,
147
+ "eval_loss": 0.2235146462917328,
148
+ "eval_runtime": 27.2082,
149
+ "eval_samples_per_second": 1016.311,
150
+ "eval_steps_per_second": 7.976,
151
+ "step": 1500
152
+ },
153
+ {
154
+ "epoch": 1.8497109826589595,
155
+ "eval_loss": 0.21805228292942047,
156
+ "eval_runtime": 26.5895,
157
+ "eval_samples_per_second": 1039.959,
158
+ "eval_steps_per_second": 8.161,
159
+ "step": 1600
160
+ },
161
+ {
162
+ "epoch": 1.9653179190751446,
163
+ "eval_loss": 0.22142848372459412,
164
+ "eval_runtime": 26.6159,
165
+ "eval_samples_per_second": 1038.929,
166
+ "eval_steps_per_second": 8.153,
167
+ "step": 1700
168
+ },
169
+ {
170
+ "epoch": 2.0809248554913293,
171
+ "eval_loss": 0.21644671261310577,
172
+ "eval_runtime": 26.2398,
173
+ "eval_samples_per_second": 1053.819,
174
+ "eval_steps_per_second": 8.27,
175
+ "step": 1800
176
+ },
177
+ {
178
+ "epoch": 2.1965317919075145,
179
+ "eval_loss": 0.2150029093027115,
180
+ "eval_runtime": 25.8402,
181
+ "eval_samples_per_second": 1070.114,
182
+ "eval_steps_per_second": 8.398,
183
+ "step": 1900
184
+ },
185
+ {
186
+ "epoch": 2.3121387283236996,
187
+ "grad_norm": 1.9824634790420532,
188
+ "learning_rate": 9.537803468208093e-05,
189
+ "loss": 0.14658987426757814,
190
+ "step": 2000
191
+ },
192
+ {
193
+ "epoch": 2.3121387283236996,
194
+ "eval_loss": 0.213293194770813,
195
+ "eval_runtime": 25.5778,
196
+ "eval_samples_per_second": 1081.094,
197
+ "eval_steps_per_second": 8.484,
198
+ "step": 2000
199
+ },
200
+ {
201
+ "epoch": 2.4277456647398843,
202
+ "eval_loss": 0.21219080686569214,
203
+ "eval_runtime": 25.765,
204
+ "eval_samples_per_second": 1073.239,
205
+ "eval_steps_per_second": 8.422,
206
+ "step": 2100
207
+ },
208
+ {
209
+ "epoch": 2.5433526011560694,
210
+ "eval_loss": 0.2117120325565338,
211
+ "eval_runtime": 26.1201,
212
+ "eval_samples_per_second": 1058.648,
213
+ "eval_steps_per_second": 8.308,
214
+ "step": 2200
215
+ },
216
+ {
217
+ "epoch": 2.6589595375722546,
218
+ "eval_loss": 0.21178804337978363,
219
+ "eval_runtime": 26.1939,
220
+ "eval_samples_per_second": 1055.664,
221
+ "eval_steps_per_second": 8.284,
222
+ "step": 2300
223
+ },
224
+ {
225
+ "epoch": 2.7745664739884393,
226
+ "eval_loss": 0.2113269418478012,
227
+ "eval_runtime": 26.5141,
228
+ "eval_samples_per_second": 1042.916,
229
+ "eval_steps_per_second": 8.184,
230
+ "step": 2400
231
+ },
232
+ {
233
+ "epoch": 2.8901734104046244,
234
+ "grad_norm": 1.6065114736557007,
235
+ "learning_rate": 9.422196531791908e-05,
236
+ "loss": 0.13519590759277345,
237
+ "step": 2500
238
+ },
239
+ {
240
+ "epoch": 2.8901734104046244,
241
+ "eval_loss": 0.20970419049263,
242
+ "eval_runtime": 26.2255,
243
+ "eval_samples_per_second": 1054.393,
244
+ "eval_steps_per_second": 8.274,
245
+ "step": 2500
246
+ },
247
+ {
248
+ "epoch": 3.005780346820809,
249
+ "eval_loss": 0.2075337916612625,
250
+ "eval_runtime": 26.4595,
251
+ "eval_samples_per_second": 1045.07,
252
+ "eval_steps_per_second": 8.201,
253
+ "step": 2600
254
+ }
255
+ ],
256
+ "logging_steps": 500,
257
+ "max_steps": 43250,
258
+ "num_input_tokens_seen": 0,
259
+ "num_train_epochs": 50,
260
+ "save_steps": 100,
261
+ "stateful_callbacks": {
262
+ "EarlyStoppingCallback": {
263
+ "args": {
264
+ "early_stopping_patience": 5,
265
+ "early_stopping_threshold": 0.0
266
+ },
267
+ "attributes": {
268
+ "early_stopping_patience_counter": 0
269
+ }
270
+ },
271
+ "TrainerControl": {
272
+ "args": {
273
+ "should_epoch_stop": false,
274
+ "should_evaluate": false,
275
+ "should_log": false,
276
+ "should_save": true,
277
+ "should_training_stop": false
278
+ },
279
+ "attributes": {}
280
+ }
281
+ },
282
+ "total_flos": 0.0,
283
+ "train_batch_size": 128,
284
+ "trial_name": null,
285
+ "trial_params": null
286
+ }
checkpoint-2600/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbaa10d98d1cdd9a980547ffbef8974da829ab33a5dd461feba1e6741bdeb5fc
3
+ size 5585
checkpoint-3000/1_Pooling/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "embedding_dimension": 384,
3
+ "pooling_mode": "mean",
4
+ "include_prompt": true
5
+ }
checkpoint-3000/2_Normalize/config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "module_input_name": "sentence_embedding",
3
+ "module_output_name": "sentence_embedding"
4
+ }
checkpoint-3000/README.md ADDED
@@ -0,0 +1,437 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - en
4
+ tags:
5
+ - sentence-transformers
6
+ - sentence-similarity
7
+ - feature-extraction
8
+ - dense
9
+ - generated_from_trainer
10
+ - dataset_size:110608
11
+ - loss:MultipleNegativesRankingLoss
12
+ base_model: sentence-transformers/all-MiniLM-L6-v2
13
+ widget:
14
+ - source_sentence: proficiency in Yiddish required for this position
15
+ sentences:
16
+ - write Telugu
17
+ - Latvian
18
+ - Yiddish
19
+ - source_sentence: knowledge of international cinema is a necessity
20
+ sentences:
21
+ - SPARK
22
+ - Italian
23
+ - film studies
24
+ - source_sentence: experience with casting anode pins is a plus.
25
+ sentences:
26
+ - 'plan evaluation '
27
+ - synthetic natural environment
28
+ - cast cell covers
29
+ - source_sentence: Native speakers of Telugu are especially encouraged to apply.
30
+ sentences:
31
+ - Romanian
32
+ - Maltego
33
+ - Telugu
34
+ - source_sentence: looking for experienced animal hunters to join our team
35
+ sentences:
36
+ - animal hunting
37
+ - use databases
38
+ - electrical engineering
39
+ datasets:
40
+ - TechWolf/Synthetic-ESCO-skill-sentences
41
+ pipeline_tag: sentence-similarity
42
+ library_name: sentence-transformers
43
+ ---
44
+
45
+ # SentenceTransformer based on sentence-transformers/all-MiniLM-L6-v2
46
+
47
+ This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) on the [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) dataset. It maps inputs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, classification, clustering, and more.
48
+
49
+ ## Model Details
50
+
51
+ ### Model Description
52
+ - **Model Type:** Sentence Transformer
53
+ - **Base model:** [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) <!-- at revision 1110a243fdf4706b3f48f1d95db1a4f5529b4d41 -->
54
+ - **Maximum Sequence Length:** 256 tokens
55
+ - **Output Dimensionality:** 384 dimensions
56
+ - **Similarity Function:** Cosine Similarity
57
+ - **Supported Modality:** Text
58
+ - **Training Dataset:**
59
+ - [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences)
60
+ - **Language:** en
61
+ <!-- - **License:** Unknown -->
62
+
63
+ ### Model Sources
64
+
65
+ - **Documentation:** [Sentence Transformers Documentation](https://sbert.net)
66
+ - **Repository:** [Sentence Transformers on GitHub](https://github.com/huggingface/sentence-transformers)
67
+ - **Hugging Face:** [Sentence Transformers on Hugging Face](https://huggingface.co/models?library=sentence-transformers)
68
+
69
+ ### Full Model Architecture
70
+
71
+ ```
72
+ SentenceTransformer(
73
+ (0): Transformer({'transformer_task': 'feature-extraction', 'modality_config': {'text': {'method': 'forward', 'method_output_name': 'last_hidden_state'}}, 'module_output_name': 'token_embeddings', 'architecture': 'BertModel'})
74
+ (1): Pooling({'embedding_dimension': 384, 'pooling_mode': 'mean', 'include_prompt': True})
75
+ (2): Normalize({'module_input_name': 'sentence_embedding', 'module_output_name': 'sentence_embedding'})
76
+ )
77
+ ```
78
+
79
+ ## Usage
80
+
81
+ ### Direct Usage (Sentence Transformers)
82
+
83
+ First install the Sentence Transformers library:
84
+
85
+ ```bash
86
+ pip install -U sentence-transformers
87
+ ```
88
+ Then you can load this model and run inference.
89
+ ```python
90
+ from sentence_transformers import SentenceTransformer
91
+
92
+ # Download from the 🤗 Hub
93
+ model = SentenceTransformer("sentence_transformers_model_id")
94
+ # Run inference
95
+ sentences = [
96
+ 'looking for experienced animal hunters to join our team',
97
+ 'animal hunting',
98
+ 'electrical engineering',
99
+ ]
100
+ embeddings = model.encode(sentences)
101
+ print(embeddings.shape)
102
+ # [3, 384]
103
+
104
+ # Get the similarity scores for the embeddings
105
+ similarities = model.similarity(embeddings, embeddings)
106
+ print(similarities)
107
+ # tensor([[ 1.0000, 0.8110, -0.1223],
108
+ # [ 0.8110, 1.0000, -0.1256],
109
+ # [-0.1223, -0.1256, 1.0000]])
110
+ ```
111
+ <!--
112
+ ### Direct Usage (Transformers)
113
+
114
+ <details><summary>Click to see the direct usage in Transformers</summary>
115
+
116
+ </details>
117
+ -->
118
+
119
+ <!--
120
+ ### Downstream Usage (Sentence Transformers)
121
+
122
+ You can finetune this model on your own dataset.
123
+
124
+ <details><summary>Click to expand</summary>
125
+
126
+ </details>
127
+ -->
128
+
129
+ <!--
130
+ ### Out-of-Scope Use
131
+
132
+ *List how the model may foreseeably be misused and address what users ought not to do with the model.*
133
+ -->
134
+
135
+ <!--
136
+ ## Bias, Risks and Limitations
137
+
138
+ *What are the known or foreseeable issues stemming from this model? You could also flag here known failure cases or weaknesses of the model.*
139
+ -->
140
+
141
+ <!--
142
+ ### Recommendations
143
+
144
+ *What are recommendations with respect to the foreseeable issues? For example, filtering explicit content.*
145
+ -->
146
+
147
+ ## Training Details
148
+
149
+ ### Training Dataset
150
+
151
+ #### synthetic-esco-skill-sentences
152
+
153
+ * Dataset: [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) at [09fc487](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences/tree/09fc487d1394c1fdfd224cb621a89b14af925716)
154
+ * Size: 110,608 training samples
155
+ * Columns: <code>anchor</code> and <code>positive</code>
156
+ * Approximate statistics based on the first 100 samples:
157
+ | | anchor | positive |
158
+ |:---------|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|
159
+ | type | string | string |
160
+ | modality | text | text |
161
+ | details | <ul><li>min: 11 tokens</li><li>mean: 19.69 tokens</li><li>max: 30 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 6.53 tokens</li><li>max: 15 tokens</li></ul> |
162
+ * Samples:
163
+ | anchor | positive |
164
+ |:-----------------------------------------------------------------------------------------------------------------|:--------------------------------------------|
165
+ | <code>Applicants should be skilled in examining legislative drafts for legal compliance and accuracy.</code> | <code>examine legislative drafts</code> |
166
+ | <code>Candidate must have knowledge in photographic film processing, including monitoring chemical baths.</code> | <code>monitor film development baths</code> |
167
+ | <code>we are looking for an experienced meat grinder to join our team</code> | <code>grind meat</code> |
168
+ * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
169
+ ```json
170
+ {
171
+ "scale": 20.0,
172
+ "similarity_fct": "cos_sim",
173
+ "gather_across_devices": false,
174
+ "directions": [
175
+ "query_to_doc"
176
+ ],
177
+ "partition_mode": "joint",
178
+ "hardness_mode": null,
179
+ "hardness_strength": 0.0
180
+ }
181
+ ```
182
+
183
+ ### Evaluation Dataset
184
+
185
+ #### synthetic-esco-skill-sentences
186
+
187
+ * Dataset: [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) at [09fc487](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences/tree/09fc487d1394c1fdfd224cb621a89b14af925716)
188
+ * Size: 27,652 evaluation samples
189
+ * Columns: <code>anchor</code> and <code>positive</code>
190
+ * Approximate statistics based on the first 100 samples:
191
+ | | anchor | positive |
192
+ |:---------|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|
193
+ | type | string | string |
194
+ | modality | text | text |
195
+ | details | <ul><li>min: 11 tokens</li><li>mean: 19.55 tokens</li><li>max: 34 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 5.96 tokens</li><li>max: 10 tokens</li></ul> |
196
+ * Samples:
197
+ | anchor | positive |
198
+ |:-----------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------|
199
+ | <code>the position requires attention to detail and strong organizational skills in order to maintain accurate records of security incidents.</code> | <code>document security incidents in the store</code> |
200
+ | <code>We need someone with experience in export logistics to lead our transportation team</code> | <code>coordinate export transportation activities</code> |
201
+ | <code>The ideal candidate will have experience in developing food policy that meets the social objectives of our company.</code> | <code>develop food policy</code> |
202
+ * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
203
+ ```json
204
+ {
205
+ "scale": 20.0,
206
+ "similarity_fct": "cos_sim",
207
+ "gather_across_devices": false,
208
+ "directions": [
209
+ "query_to_doc"
210
+ ],
211
+ "partition_mode": "joint",
212
+ "hardness_mode": null,
213
+ "hardness_strength": 0.0
214
+ }
215
+ ```
216
+
217
+ ### Training Hyperparameters
218
+ #### Non-Default Hyperparameters
219
+
220
+ - `per_device_train_batch_size`: 128
221
+ - `num_train_epochs`: 50
222
+ - `learning_rate`: 0.0001
223
+ - `weight_decay`: 0.01
224
+ - `per_device_eval_batch_size`: 128
225
+ - `dataloader_pin_memory`: False
226
+
227
+ #### All Hyperparameters
228
+ <details><summary>Click to expand</summary>
229
+
230
+ - `per_device_train_batch_size`: 128
231
+ - `num_train_epochs`: 50
232
+ - `max_steps`: -1
233
+ - `learning_rate`: 0.0001
234
+ - `lr_scheduler_type`: linear
235
+ - `lr_scheduler_kwargs`: None
236
+ - `warmup_steps`: 0
237
+ - `optim`: adamw_torch_fused
238
+ - `optim_args`: None
239
+ - `weight_decay`: 0.01
240
+ - `adam_beta1`: 0.9
241
+ - `adam_beta2`: 0.999
242
+ - `adam_epsilon`: 1e-08
243
+ - `optim_target_modules`: None
244
+ - `gradient_accumulation_steps`: 1
245
+ - `average_tokens_across_devices`: True
246
+ - `max_grad_norm`: 1.0
247
+ - `label_smoothing_factor`: 0.0
248
+ - `bf16`: False
249
+ - `fp16`: False
250
+ - `bf16_full_eval`: False
251
+ - `fp16_full_eval`: False
252
+ - `tf32`: None
253
+ - `gradient_checkpointing`: False
254
+ - `gradient_checkpointing_kwargs`: None
255
+ - `torch_compile`: False
256
+ - `torch_compile_backend`: None
257
+ - `torch_compile_mode`: None
258
+ - `use_liger_kernel`: False
259
+ - `liger_kernel_config`: None
260
+ - `use_cache`: False
261
+ - `neftune_noise_alpha`: None
262
+ - `torch_empty_cache_steps`: None
263
+ - `auto_find_batch_size`: False
264
+ - `log_on_each_node`: True
265
+ - `logging_nan_inf_filter`: True
266
+ - `include_num_input_tokens_seen`: no
267
+ - `log_level`: passive
268
+ - `log_level_replica`: warning
269
+ - `disable_tqdm`: False
270
+ - `project`: huggingface
271
+ - `trackio_space_id`: None
272
+ - `trackio_bucket_id`: None
273
+ - `trackio_static_space_id`: None
274
+ - `per_device_eval_batch_size`: 128
275
+ - `prediction_loss_only`: True
276
+ - `eval_on_start`: False
277
+ - `eval_do_concat_batches`: True
278
+ - `eval_use_gather_object`: False
279
+ - `eval_accumulation_steps`: None
280
+ - `include_for_metrics`: []
281
+ - `batch_eval_metrics`: False
282
+ - `save_only_model`: False
283
+ - `save_on_each_node`: False
284
+ - `enable_jit_checkpoint`: False
285
+ - `push_to_hub`: False
286
+ - `hub_private_repo`: None
287
+ - `hub_model_id`: None
288
+ - `hub_strategy`: every_save
289
+ - `hub_always_push`: False
290
+ - `hub_revision`: None
291
+ - `load_best_model_at_end`: False
292
+ - `ignore_data_skip`: False
293
+ - `restore_callback_states_from_checkpoint`: False
294
+ - `full_determinism`: False
295
+ - `seed`: 42
296
+ - `data_seed`: None
297
+ - `use_cpu`: False
298
+ - `accelerator_config`: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
299
+ - `parallelism_config`: None
300
+ - `dataloader_drop_last`: False
301
+ - `dataloader_num_workers`: 0
302
+ - `dataloader_pin_memory`: False
303
+ - `dataloader_persistent_workers`: False
304
+ - `dataloader_prefetch_factor`: None
305
+ - `dataloader_multiprocessing_context`: None
306
+ - `dataloader_in_order`: True
307
+ - `remove_unused_columns`: True
308
+ - `label_names`: None
309
+ - `train_sampling_strategy`: random
310
+ - `length_column_name`: length
311
+ - `ddp_find_unused_parameters`: None
312
+ - `ddp_bucket_cap_mb`: None
313
+ - `ddp_broadcast_buffers`: False
314
+ - `ddp_static_graph`: None
315
+ - `ddp_backend`: None
316
+ - `ddp_timeout`: 1800
317
+ - `fsdp`: None
318
+ - `fsdp_config`: None
319
+ - `deepspeed`: None
320
+ - `debug`: []
321
+ - `skip_memory_metrics`: True
322
+ - `do_predict`: False
323
+ - `resume_from_checkpoint`: None
324
+ - `local_rank`: -1
325
+ - `prompts`: None
326
+ - `batch_sampler`: batch_sampler
327
+ - `multi_dataset_batch_sampler`: proportional
328
+ - `router_mapping`: {}
329
+ - `learning_rate_mapping`: {}
330
+ - `warmup_ratio`: None
331
+
332
+ </details>
333
+
334
+ ### Training Logs
335
+ | Epoch | Step | Training Loss | Validation Loss |
336
+ |:------:|:----:|:-------------:|:---------------:|
337
+ | 0.1156 | 100 | - | 0.2757 |
338
+ | 0.2312 | 200 | - | 0.2636 |
339
+ | 0.3468 | 300 | - | 0.2548 |
340
+ | 0.4624 | 400 | - | 0.2484 |
341
+ | 0.5780 | 500 | 0.2929 | 0.2452 |
342
+ | 0.6936 | 600 | - | 0.2387 |
343
+ | 0.8092 | 700 | - | 0.2291 |
344
+ | 0.9249 | 800 | - | 0.2355 |
345
+ | 1.0405 | 900 | - | 0.2288 |
346
+ | 1.1561 | 1000 | 0.2378 | 0.2275 |
347
+ | 1.2717 | 1100 | - | 0.2263 |
348
+ | 1.3873 | 1200 | - | 0.2282 |
349
+ | 1.5029 | 1300 | - | 0.2228 |
350
+ | 1.6185 | 1400 | - | 0.2199 |
351
+ | 1.7341 | 1500 | 0.1797 | 0.2235 |
352
+ | 1.8497 | 1600 | - | 0.2181 |
353
+ | 1.9653 | 1700 | - | 0.2214 |
354
+ | 2.0809 | 1800 | - | 0.2164 |
355
+ | 2.1965 | 1900 | - | 0.2150 |
356
+ | 2.3121 | 2000 | 0.1466 | 0.2133 |
357
+ | 2.4277 | 2100 | - | 0.2122 |
358
+ | 2.5434 | 2200 | - | 0.2117 |
359
+ | 2.6590 | 2300 | - | 0.2118 |
360
+ | 2.7746 | 2400 | - | 0.2113 |
361
+ | 2.8902 | 2500 | 0.1352 | 0.2097 |
362
+ | 3.0058 | 2600 | - | 0.2075 |
363
+ | 3.1214 | 2700 | - | 0.2110 |
364
+ | 3.2370 | 2800 | - | 0.2114 |
365
+ | 3.3526 | 2900 | - | 0.2170 |
366
+ | 3.4682 | 3000 | 0.1012 | 0.2143 |
367
+
368
+
369
+ ### Training Time
370
+ - **Training**: 20.2 minutes
371
+ - **Evaluation**: 12.9 minutes
372
+ - **Total**: 33.1 minutes
373
+
374
+ ### Framework Versions
375
+ - Python: 3.14.7
376
+ - Sentence Transformers: 6.2.0
377
+ - Transformers: 5.22.0
378
+ - PyTorch: 2.15.0
379
+ - Accelerate: 1.15.0
380
+ - Datasets: 5.1.0
381
+ - Tokenizers: 0.23.3
382
+
383
+ ## Additional Resources
384
+
385
+ - [Training and Finetuning Embedding Models with Sentence Transformers](https://huggingface.co/blog/train-sentence-transformers): the end-to-end guide for training or finetuning Sentence Transformer models.
386
+ - [Introduction to Matryoshka Embedding Models](https://huggingface.co/blog/matryoshka): variable-size embeddings that can be truncated with minimal quality loss.
387
+ - [Binary and Scalar Embedding Quantization for Significantly Faster & Cheaper Retrieval](https://huggingface.co/blog/embedding-quantization): post-training compression of embedding vectors.
388
+ - [Multimodal Embedding & Reranker Models with Sentence Transformers](https://huggingface.co/blog/multimodal-sentence-transformers): use text, image, audio, and video models through the same API.
389
+ - [Training and Finetuning Multimodal Embedding & Reranker Models with Sentence Transformers](https://huggingface.co/blog/train-multimodal-sentence-transformers): train multimodal embedding models, with a Visual Document Retrieval walkthrough.
390
+
391
+ ## Citation
392
+
393
+ ### BibTeX
394
+
395
+ #### Sentence Transformers
396
+ ```bibtex
397
+ @inproceedings{reimers-2019-sentence-bert,
398
+ title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
399
+ author = "Reimers, Nils and Gurevych, Iryna",
400
+ booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
401
+ month = "11",
402
+ year = "2019",
403
+ publisher = "Association for Computational Linguistics",
404
+ url = "https://arxiv.org/abs/1908.10084",
405
+ }
406
+ ```
407
+
408
+ #### MultipleNegativesRankingLoss
409
+ ```bibtex
410
+ @misc{oord2019representationlearningcontrastivepredictive,
411
+ title={Representation Learning with Contrastive Predictive Coding},
412
+ author={Aaron van den Oord and Yazhe Li and Oriol Vinyals},
413
+ year={2019},
414
+ eprint={1807.03748},
415
+ archivePrefix={arXiv},
416
+ primaryClass={cs.LG},
417
+ url={https://arxiv.org/abs/1807.03748},
418
+ }
419
+ ```
420
+
421
+ <!--
422
+ ## Glossary
423
+
424
+ *Clearly define terms in order to be accessible across audiences.*
425
+ -->
426
+
427
+ <!--
428
+ ## Model Card Authors
429
+
430
+ *Lists the people who create the model card, providing recognition and accountability for the detailed work that goes into its construction.*
431
+ -->
432
+
433
+ <!--
434
+ ## Model Card Contact
435
+
436
+ *Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors.*
437
+ -->
checkpoint-3000/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_cross_attention": false,
3
+ "architectures": [
4
+ "BertModel"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": null,
8
+ "classifier_dropout": null,
9
+ "dtype": "float32",
10
+ "eos_token_id": null,
11
+ "gradient_checkpointing": false,
12
+ "hidden_act": "gelu",
13
+ "hidden_dropout_prob": 0.1,
14
+ "hidden_size": 384,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 1536,
17
+ "is_decoder": false,
18
+ "layer_norm_eps": 1e-12,
19
+ "max_position_embeddings": 512,
20
+ "model_type": "bert",
21
+ "num_attention_heads": 12,
22
+ "num_hidden_layers": 6,
23
+ "pad_token_id": 0,
24
+ "position_embedding_type": "absolute",
25
+ "tie_word_embeddings": true,
26
+ "transformers_version": "5.22.0",
27
+ "type_vocab_size": 2,
28
+ "use_cache": false,
29
+ "vocab_size": 30522
30
+ }
checkpoint-3000/config_sentence_transformers.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "__version__": {
3
+ "pytorch": "2.15.0",
4
+ "sentence_transformers": "6.2.0",
5
+ "transformers": "5.22.0"
6
+ },
7
+ "default_prompt_name": null,
8
+ "model_type": "SentenceTransformer",
9
+ "prompts": {
10
+ "document": "",
11
+ "query": ""
12
+ },
13
+ "similarity_fn_name": "cosine"
14
+ }
checkpoint-3000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:284709298418f4634a1ebfd37eed6876f467b76e6658ce29de8a793c1feea3b3
3
+ size 90864192
checkpoint-3000/modules.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "idx": 0,
4
+ "name": "0",
5
+ "path": "",
6
+ "type": "sentence_transformers.base.modules.transformer.Transformer"
7
+ },
8
+ {
9
+ "idx": 1,
10
+ "name": "1",
11
+ "path": "1_Pooling",
12
+ "type": "sentence_transformers.sentence_transformer.modules.pooling.Pooling"
13
+ },
14
+ {
15
+ "idx": 2,
16
+ "name": "2",
17
+ "path": "2_Normalize",
18
+ "type": "sentence_transformers.base.modules.normalize.Normalize"
19
+ }
20
+ ]
checkpoint-3000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ab701f6267448ff2cbf7f8aa4598b4bfd51dc4ea9379ff33dcdd5115e09340b
3
+ size 180605387
checkpoint-3000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32b7919406ffdba09cff98a19dda0eb3f3d3d6b79e54984b18f62a8d5ae6d762
3
+ size 14391
checkpoint-3000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab6169451be2876f79086d0a551067f805fd7b8bb80ed461e3f15e7c68942525
3
+ size 1465
checkpoint-3000/sentence_bert_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "transformer_task": "feature-extraction",
3
+ "modality_config": {
4
+ "text": {
5
+ "method": "forward",
6
+ "method_output_name": "last_hidden_state"
7
+ }
8
+ },
9
+ "module_output_name": "token_embeddings"
10
+ }
checkpoint-3000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-3000/tokenizer_config.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "cls_token": "[CLS]",
4
+ "do_basic_tokenize": true,
5
+ "do_lower_case": true,
6
+ "is_local": false,
7
+ "local_files_only": false,
8
+ "mask_token": "[MASK]",
9
+ "max_length": 128,
10
+ "model_max_length": 256,
11
+ "never_split": null,
12
+ "pad_to_multiple_of": null,
13
+ "pad_token": "[PAD]",
14
+ "pad_token_type_id": 0,
15
+ "padding_side": "right",
16
+ "sep_token": "[SEP]",
17
+ "stride": 0,
18
+ "strip_accents": null,
19
+ "tokenize_chinese_chars": true,
20
+ "tokenizer_class": "BertTokenizer",
21
+ "truncation_side": "right",
22
+ "truncation_strategy": "longest_first",
23
+ "unk_token": "[UNK]"
24
+ }
checkpoint-3000/trainer_state.json ADDED
@@ -0,0 +1,325 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 2600,
3
+ "best_metric": 0.2075337916612625,
4
+ "best_model_checkpoint": "ft/sentence-transformers/all-MiniLM-L6-v2/checkpoint-2600",
5
+ "epoch": 3.468208092485549,
6
+ "eval_steps": 100,
7
+ "global_step": 3000,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.11560693641618497,
14
+ "eval_loss": 0.2756878137588501,
15
+ "eval_runtime": 19.195,
16
+ "eval_samples_per_second": 1440.58,
17
+ "eval_steps_per_second": 11.305,
18
+ "step": 100
19
+ },
20
+ {
21
+ "epoch": 0.23121387283236994,
22
+ "eval_loss": 0.2635912597179413,
23
+ "eval_runtime": 19.1881,
24
+ "eval_samples_per_second": 1441.105,
25
+ "eval_steps_per_second": 11.309,
26
+ "step": 200
27
+ },
28
+ {
29
+ "epoch": 0.3468208092485549,
30
+ "eval_loss": 0.25484976172447205,
31
+ "eval_runtime": 20.4586,
32
+ "eval_samples_per_second": 1351.608,
33
+ "eval_steps_per_second": 10.607,
34
+ "step": 300
35
+ },
36
+ {
37
+ "epoch": 0.4624277456647399,
38
+ "eval_loss": 0.24843913316726685,
39
+ "eval_runtime": 24.2538,
40
+ "eval_samples_per_second": 1140.109,
41
+ "eval_steps_per_second": 8.947,
42
+ "step": 400
43
+ },
44
+ {
45
+ "epoch": 0.5780346820809249,
46
+ "grad_norm": 3.031534194946289,
47
+ "learning_rate": 9.884624277456648e-05,
48
+ "loss": 0.2928614196777344,
49
+ "step": 500
50
+ },
51
+ {
52
+ "epoch": 0.5780346820809249,
53
+ "eval_loss": 0.24522048234939575,
54
+ "eval_runtime": 24.3615,
55
+ "eval_samples_per_second": 1135.067,
56
+ "eval_steps_per_second": 8.907,
57
+ "step": 500
58
+ },
59
+ {
60
+ "epoch": 0.6936416184971098,
61
+ "eval_loss": 0.23871970176696777,
62
+ "eval_runtime": 23.7379,
63
+ "eval_samples_per_second": 1164.886,
64
+ "eval_steps_per_second": 9.141,
65
+ "step": 600
66
+ },
67
+ {
68
+ "epoch": 0.8092485549132948,
69
+ "eval_loss": 0.22912940382957458,
70
+ "eval_runtime": 23.4904,
71
+ "eval_samples_per_second": 1177.162,
72
+ "eval_steps_per_second": 9.238,
73
+ "step": 700
74
+ },
75
+ {
76
+ "epoch": 0.9248554913294798,
77
+ "eval_loss": 0.23552891612052917,
78
+ "eval_runtime": 23.3878,
79
+ "eval_samples_per_second": 1182.328,
80
+ "eval_steps_per_second": 9.278,
81
+ "step": 800
82
+ },
83
+ {
84
+ "epoch": 1.0404624277456647,
85
+ "eval_loss": 0.22875414788722992,
86
+ "eval_runtime": 24.2858,
87
+ "eval_samples_per_second": 1138.608,
88
+ "eval_steps_per_second": 8.935,
89
+ "step": 900
90
+ },
91
+ {
92
+ "epoch": 1.1560693641618498,
93
+ "grad_norm": 2.5727860927581787,
94
+ "learning_rate": 9.769017341040463e-05,
95
+ "loss": 0.2378170166015625,
96
+ "step": 1000
97
+ },
98
+ {
99
+ "epoch": 1.1560693641618498,
100
+ "eval_loss": 0.22745409607887268,
101
+ "eval_runtime": 24.6958,
102
+ "eval_samples_per_second": 1119.705,
103
+ "eval_steps_per_second": 8.787,
104
+ "step": 1000
105
+ },
106
+ {
107
+ "epoch": 1.2716763005780347,
108
+ "eval_loss": 0.22631044685840607,
109
+ "eval_runtime": 25.1507,
110
+ "eval_samples_per_second": 1099.453,
111
+ "eval_steps_per_second": 8.628,
112
+ "step": 1100
113
+ },
114
+ {
115
+ "epoch": 1.3872832369942196,
116
+ "eval_loss": 0.22824475169181824,
117
+ "eval_runtime": 25.1355,
118
+ "eval_samples_per_second": 1100.116,
119
+ "eval_steps_per_second": 8.633,
120
+ "step": 1200
121
+ },
122
+ {
123
+ "epoch": 1.5028901734104045,
124
+ "eval_loss": 0.22276242077350616,
125
+ "eval_runtime": 25.2896,
126
+ "eval_samples_per_second": 1093.414,
127
+ "eval_steps_per_second": 8.581,
128
+ "step": 1300
129
+ },
130
+ {
131
+ "epoch": 1.6184971098265897,
132
+ "eval_loss": 0.21992968022823334,
133
+ "eval_runtime": 25.7239,
134
+ "eval_samples_per_second": 1074.952,
135
+ "eval_steps_per_second": 8.436,
136
+ "step": 1400
137
+ },
138
+ {
139
+ "epoch": 1.7341040462427746,
140
+ "grad_norm": 2.2508647441864014,
141
+ "learning_rate": 9.653410404624278e-05,
142
+ "loss": 0.1796602783203125,
143
+ "step": 1500
144
+ },
145
+ {
146
+ "epoch": 1.7341040462427746,
147
+ "eval_loss": 0.2235146462917328,
148
+ "eval_runtime": 27.2082,
149
+ "eval_samples_per_second": 1016.311,
150
+ "eval_steps_per_second": 7.976,
151
+ "step": 1500
152
+ },
153
+ {
154
+ "epoch": 1.8497109826589595,
155
+ "eval_loss": 0.21805228292942047,
156
+ "eval_runtime": 26.5895,
157
+ "eval_samples_per_second": 1039.959,
158
+ "eval_steps_per_second": 8.161,
159
+ "step": 1600
160
+ },
161
+ {
162
+ "epoch": 1.9653179190751446,
163
+ "eval_loss": 0.22142848372459412,
164
+ "eval_runtime": 26.6159,
165
+ "eval_samples_per_second": 1038.929,
166
+ "eval_steps_per_second": 8.153,
167
+ "step": 1700
168
+ },
169
+ {
170
+ "epoch": 2.0809248554913293,
171
+ "eval_loss": 0.21644671261310577,
172
+ "eval_runtime": 26.2398,
173
+ "eval_samples_per_second": 1053.819,
174
+ "eval_steps_per_second": 8.27,
175
+ "step": 1800
176
+ },
177
+ {
178
+ "epoch": 2.1965317919075145,
179
+ "eval_loss": 0.2150029093027115,
180
+ "eval_runtime": 25.8402,
181
+ "eval_samples_per_second": 1070.114,
182
+ "eval_steps_per_second": 8.398,
183
+ "step": 1900
184
+ },
185
+ {
186
+ "epoch": 2.3121387283236996,
187
+ "grad_norm": 1.9824634790420532,
188
+ "learning_rate": 9.537803468208093e-05,
189
+ "loss": 0.14658987426757814,
190
+ "step": 2000
191
+ },
192
+ {
193
+ "epoch": 2.3121387283236996,
194
+ "eval_loss": 0.213293194770813,
195
+ "eval_runtime": 25.5778,
196
+ "eval_samples_per_second": 1081.094,
197
+ "eval_steps_per_second": 8.484,
198
+ "step": 2000
199
+ },
200
+ {
201
+ "epoch": 2.4277456647398843,
202
+ "eval_loss": 0.21219080686569214,
203
+ "eval_runtime": 25.765,
204
+ "eval_samples_per_second": 1073.239,
205
+ "eval_steps_per_second": 8.422,
206
+ "step": 2100
207
+ },
208
+ {
209
+ "epoch": 2.5433526011560694,
210
+ "eval_loss": 0.2117120325565338,
211
+ "eval_runtime": 26.1201,
212
+ "eval_samples_per_second": 1058.648,
213
+ "eval_steps_per_second": 8.308,
214
+ "step": 2200
215
+ },
216
+ {
217
+ "epoch": 2.6589595375722546,
218
+ "eval_loss": 0.21178804337978363,
219
+ "eval_runtime": 26.1939,
220
+ "eval_samples_per_second": 1055.664,
221
+ "eval_steps_per_second": 8.284,
222
+ "step": 2300
223
+ },
224
+ {
225
+ "epoch": 2.7745664739884393,
226
+ "eval_loss": 0.2113269418478012,
227
+ "eval_runtime": 26.5141,
228
+ "eval_samples_per_second": 1042.916,
229
+ "eval_steps_per_second": 8.184,
230
+ "step": 2400
231
+ },
232
+ {
233
+ "epoch": 2.8901734104046244,
234
+ "grad_norm": 1.6065114736557007,
235
+ "learning_rate": 9.422196531791908e-05,
236
+ "loss": 0.13519590759277345,
237
+ "step": 2500
238
+ },
239
+ {
240
+ "epoch": 2.8901734104046244,
241
+ "eval_loss": 0.20970419049263,
242
+ "eval_runtime": 26.2255,
243
+ "eval_samples_per_second": 1054.393,
244
+ "eval_steps_per_second": 8.274,
245
+ "step": 2500
246
+ },
247
+ {
248
+ "epoch": 3.005780346820809,
249
+ "eval_loss": 0.2075337916612625,
250
+ "eval_runtime": 26.4595,
251
+ "eval_samples_per_second": 1045.07,
252
+ "eval_steps_per_second": 8.201,
253
+ "step": 2600
254
+ },
255
+ {
256
+ "epoch": 3.121387283236994,
257
+ "eval_loss": 0.21102772653102875,
258
+ "eval_runtime": 26.5553,
259
+ "eval_samples_per_second": 1041.298,
260
+ "eval_steps_per_second": 8.172,
261
+ "step": 2700
262
+ },
263
+ {
264
+ "epoch": 3.2369942196531793,
265
+ "eval_loss": 0.21144026517868042,
266
+ "eval_runtime": 28.0952,
267
+ "eval_samples_per_second": 984.224,
268
+ "eval_steps_per_second": 7.724,
269
+ "step": 2800
270
+ },
271
+ {
272
+ "epoch": 3.352601156069364,
273
+ "eval_loss": 0.21704500913619995,
274
+ "eval_runtime": 47.621,
275
+ "eval_samples_per_second": 580.669,
276
+ "eval_steps_per_second": 4.557,
277
+ "step": 2900
278
+ },
279
+ {
280
+ "epoch": 3.468208092485549,
281
+ "grad_norm": 1.3753323554992676,
282
+ "learning_rate": 9.306589595375723e-05,
283
+ "loss": 0.10123191833496094,
284
+ "step": 3000
285
+ },
286
+ {
287
+ "epoch": 3.468208092485549,
288
+ "eval_loss": 0.21425534784793854,
289
+ "eval_runtime": 26.2365,
290
+ "eval_samples_per_second": 1053.951,
291
+ "eval_steps_per_second": 8.271,
292
+ "step": 3000
293
+ }
294
+ ],
295
+ "logging_steps": 500,
296
+ "max_steps": 43250,
297
+ "num_input_tokens_seen": 0,
298
+ "num_train_epochs": 50,
299
+ "save_steps": 100,
300
+ "stateful_callbacks": {
301
+ "EarlyStoppingCallback": {
302
+ "args": {
303
+ "early_stopping_patience": 5,
304
+ "early_stopping_threshold": 0.0
305
+ },
306
+ "attributes": {
307
+ "early_stopping_patience_counter": 4
308
+ }
309
+ },
310
+ "TrainerControl": {
311
+ "args": {
312
+ "should_epoch_stop": false,
313
+ "should_evaluate": false,
314
+ "should_log": false,
315
+ "should_save": true,
316
+ "should_training_stop": false
317
+ },
318
+ "attributes": {}
319
+ }
320
+ },
321
+ "total_flos": 0.0,
322
+ "train_batch_size": 128,
323
+ "trial_name": null,
324
+ "trial_params": null
325
+ }
checkpoint-3000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbaa10d98d1cdd9a980547ffbef8974da829ab33a5dd461feba1e6741bdeb5fc
3
+ size 5585
checkpoint-3100/1_Pooling/config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "embedding_dimension": 384,
3
+ "pooling_mode": "mean",
4
+ "include_prompt": true
5
+ }
checkpoint-3100/2_Normalize/config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "module_input_name": "sentence_embedding",
3
+ "module_output_name": "sentence_embedding"
4
+ }
checkpoint-3100/README.md ADDED
@@ -0,0 +1,438 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - en
4
+ tags:
5
+ - sentence-transformers
6
+ - sentence-similarity
7
+ - feature-extraction
8
+ - dense
9
+ - generated_from_trainer
10
+ - dataset_size:110608
11
+ - loss:MultipleNegativesRankingLoss
12
+ base_model: sentence-transformers/all-MiniLM-L6-v2
13
+ widget:
14
+ - source_sentence: proficiency in Yiddish required for this position
15
+ sentences:
16
+ - write Telugu
17
+ - Latvian
18
+ - Yiddish
19
+ - source_sentence: knowledge of international cinema is a necessity
20
+ sentences:
21
+ - SPARK
22
+ - Italian
23
+ - film studies
24
+ - source_sentence: experience with casting anode pins is a plus.
25
+ sentences:
26
+ - 'plan evaluation '
27
+ - synthetic natural environment
28
+ - cast cell covers
29
+ - source_sentence: Native speakers of Telugu are especially encouraged to apply.
30
+ sentences:
31
+ - Romanian
32
+ - Maltego
33
+ - Telugu
34
+ - source_sentence: looking for experienced animal hunters to join our team
35
+ sentences:
36
+ - animal hunting
37
+ - use databases
38
+ - electrical engineering
39
+ datasets:
40
+ - TechWolf/Synthetic-ESCO-skill-sentences
41
+ pipeline_tag: sentence-similarity
42
+ library_name: sentence-transformers
43
+ ---
44
+
45
+ # SentenceTransformer based on sentence-transformers/all-MiniLM-L6-v2
46
+
47
+ This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) on the [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) dataset. It maps inputs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, classification, clustering, and more.
48
+
49
+ ## Model Details
50
+
51
+ ### Model Description
52
+ - **Model Type:** Sentence Transformer
53
+ - **Base model:** [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) <!-- at revision 1110a243fdf4706b3f48f1d95db1a4f5529b4d41 -->
54
+ - **Maximum Sequence Length:** 256 tokens
55
+ - **Output Dimensionality:** 384 dimensions
56
+ - **Similarity Function:** Cosine Similarity
57
+ - **Supported Modality:** Text
58
+ - **Training Dataset:**
59
+ - [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences)
60
+ - **Language:** en
61
+ <!-- - **License:** Unknown -->
62
+
63
+ ### Model Sources
64
+
65
+ - **Documentation:** [Sentence Transformers Documentation](https://sbert.net)
66
+ - **Repository:** [Sentence Transformers on GitHub](https://github.com/huggingface/sentence-transformers)
67
+ - **Hugging Face:** [Sentence Transformers on Hugging Face](https://huggingface.co/models?library=sentence-transformers)
68
+
69
+ ### Full Model Architecture
70
+
71
+ ```
72
+ SentenceTransformer(
73
+ (0): Transformer({'transformer_task': 'feature-extraction', 'modality_config': {'text': {'method': 'forward', 'method_output_name': 'last_hidden_state'}}, 'module_output_name': 'token_embeddings', 'architecture': 'BertModel'})
74
+ (1): Pooling({'embedding_dimension': 384, 'pooling_mode': 'mean', 'include_prompt': True})
75
+ (2): Normalize({'module_input_name': 'sentence_embedding', 'module_output_name': 'sentence_embedding'})
76
+ )
77
+ ```
78
+
79
+ ## Usage
80
+
81
+ ### Direct Usage (Sentence Transformers)
82
+
83
+ First install the Sentence Transformers library:
84
+
85
+ ```bash
86
+ pip install -U sentence-transformers
87
+ ```
88
+ Then you can load this model and run inference.
89
+ ```python
90
+ from sentence_transformers import SentenceTransformer
91
+
92
+ # Download from the 🤗 Hub
93
+ model = SentenceTransformer("sentence_transformers_model_id")
94
+ # Run inference
95
+ sentences = [
96
+ 'looking for experienced animal hunters to join our team',
97
+ 'animal hunting',
98
+ 'electrical engineering',
99
+ ]
100
+ embeddings = model.encode(sentences)
101
+ print(embeddings.shape)
102
+ # [3, 384]
103
+
104
+ # Get the similarity scores for the embeddings
105
+ similarities = model.similarity(embeddings, embeddings)
106
+ print(similarities)
107
+ # tensor([[ 1.0000, 0.8278, -0.1120],
108
+ # [ 0.8278, 1.0000, -0.1666],
109
+ # [-0.1120, -0.1666, 1.0000]])
110
+ ```
111
+ <!--
112
+ ### Direct Usage (Transformers)
113
+
114
+ <details><summary>Click to see the direct usage in Transformers</summary>
115
+
116
+ </details>
117
+ -->
118
+
119
+ <!--
120
+ ### Downstream Usage (Sentence Transformers)
121
+
122
+ You can finetune this model on your own dataset.
123
+
124
+ <details><summary>Click to expand</summary>
125
+
126
+ </details>
127
+ -->
128
+
129
+ <!--
130
+ ### Out-of-Scope Use
131
+
132
+ *List how the model may foreseeably be misused and address what users ought not to do with the model.*
133
+ -->
134
+
135
+ <!--
136
+ ## Bias, Risks and Limitations
137
+
138
+ *What are the known or foreseeable issues stemming from this model? You could also flag here known failure cases or weaknesses of the model.*
139
+ -->
140
+
141
+ <!--
142
+ ### Recommendations
143
+
144
+ *What are recommendations with respect to the foreseeable issues? For example, filtering explicit content.*
145
+ -->
146
+
147
+ ## Training Details
148
+
149
+ ### Training Dataset
150
+
151
+ #### synthetic-esco-skill-sentences
152
+
153
+ * Dataset: [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) at [09fc487](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences/tree/09fc487d1394c1fdfd224cb621a89b14af925716)
154
+ * Size: 110,608 training samples
155
+ * Columns: <code>anchor</code> and <code>positive</code>
156
+ * Approximate statistics based on the first 100 samples:
157
+ | | anchor | positive |
158
+ |:---------|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|
159
+ | type | string | string |
160
+ | modality | text | text |
161
+ | details | <ul><li>min: 11 tokens</li><li>mean: 19.69 tokens</li><li>max: 30 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 6.53 tokens</li><li>max: 15 tokens</li></ul> |
162
+ * Samples:
163
+ | anchor | positive |
164
+ |:-----------------------------------------------------------------------------------------------------------------|:--------------------------------------------|
165
+ | <code>Applicants should be skilled in examining legislative drafts for legal compliance and accuracy.</code> | <code>examine legislative drafts</code> |
166
+ | <code>Candidate must have knowledge in photographic film processing, including monitoring chemical baths.</code> | <code>monitor film development baths</code> |
167
+ | <code>we are looking for an experienced meat grinder to join our team</code> | <code>grind meat</code> |
168
+ * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
169
+ ```json
170
+ {
171
+ "scale": 20.0,
172
+ "similarity_fct": "cos_sim",
173
+ "gather_across_devices": false,
174
+ "directions": [
175
+ "query_to_doc"
176
+ ],
177
+ "partition_mode": "joint",
178
+ "hardness_mode": null,
179
+ "hardness_strength": 0.0
180
+ }
181
+ ```
182
+
183
+ ### Evaluation Dataset
184
+
185
+ #### synthetic-esco-skill-sentences
186
+
187
+ * Dataset: [synthetic-esco-skill-sentences](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences) at [09fc487](https://huggingface.co/datasets/TechWolf/Synthetic-ESCO-skill-sentences/tree/09fc487d1394c1fdfd224cb621a89b14af925716)
188
+ * Size: 27,652 evaluation samples
189
+ * Columns: <code>anchor</code> and <code>positive</code>
190
+ * Approximate statistics based on the first 100 samples:
191
+ | | anchor | positive |
192
+ |:---------|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|
193
+ | type | string | string |
194
+ | modality | text | text |
195
+ | details | <ul><li>min: 11 tokens</li><li>mean: 19.55 tokens</li><li>max: 34 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 5.96 tokens</li><li>max: 10 tokens</li></ul> |
196
+ * Samples:
197
+ | anchor | positive |
198
+ |:-----------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------|
199
+ | <code>the position requires attention to detail and strong organizational skills in order to maintain accurate records of security incidents.</code> | <code>document security incidents in the store</code> |
200
+ | <code>We need someone with experience in export logistics to lead our transportation team</code> | <code>coordinate export transportation activities</code> |
201
+ | <code>The ideal candidate will have experience in developing food policy that meets the social objectives of our company.</code> | <code>develop food policy</code> |
202
+ * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
203
+ ```json
204
+ {
205
+ "scale": 20.0,
206
+ "similarity_fct": "cos_sim",
207
+ "gather_across_devices": false,
208
+ "directions": [
209
+ "query_to_doc"
210
+ ],
211
+ "partition_mode": "joint",
212
+ "hardness_mode": null,
213
+ "hardness_strength": 0.0
214
+ }
215
+ ```
216
+
217
+ ### Training Hyperparameters
218
+ #### Non-Default Hyperparameters
219
+
220
+ - `per_device_train_batch_size`: 128
221
+ - `num_train_epochs`: 50
222
+ - `learning_rate`: 0.0001
223
+ - `weight_decay`: 0.01
224
+ - `per_device_eval_batch_size`: 128
225
+ - `dataloader_pin_memory`: False
226
+
227
+ #### All Hyperparameters
228
+ <details><summary>Click to expand</summary>
229
+
230
+ - `per_device_train_batch_size`: 128
231
+ - `num_train_epochs`: 50
232
+ - `max_steps`: -1
233
+ - `learning_rate`: 0.0001
234
+ - `lr_scheduler_type`: linear
235
+ - `lr_scheduler_kwargs`: None
236
+ - `warmup_steps`: 0
237
+ - `optim`: adamw_torch_fused
238
+ - `optim_args`: None
239
+ - `weight_decay`: 0.01
240
+ - `adam_beta1`: 0.9
241
+ - `adam_beta2`: 0.999
242
+ - `adam_epsilon`: 1e-08
243
+ - `optim_target_modules`: None
244
+ - `gradient_accumulation_steps`: 1
245
+ - `average_tokens_across_devices`: True
246
+ - `max_grad_norm`: 1.0
247
+ - `label_smoothing_factor`: 0.0
248
+ - `bf16`: False
249
+ - `fp16`: False
250
+ - `bf16_full_eval`: False
251
+ - `fp16_full_eval`: False
252
+ - `tf32`: None
253
+ - `gradient_checkpointing`: False
254
+ - `gradient_checkpointing_kwargs`: None
255
+ - `torch_compile`: False
256
+ - `torch_compile_backend`: None
257
+ - `torch_compile_mode`: None
258
+ - `use_liger_kernel`: False
259
+ - `liger_kernel_config`: None
260
+ - `use_cache`: False
261
+ - `neftune_noise_alpha`: None
262
+ - `torch_empty_cache_steps`: None
263
+ - `auto_find_batch_size`: False
264
+ - `log_on_each_node`: True
265
+ - `logging_nan_inf_filter`: True
266
+ - `include_num_input_tokens_seen`: no
267
+ - `log_level`: passive
268
+ - `log_level_replica`: warning
269
+ - `disable_tqdm`: False
270
+ - `project`: huggingface
271
+ - `trackio_space_id`: None
272
+ - `trackio_bucket_id`: None
273
+ - `trackio_static_space_id`: None
274
+ - `per_device_eval_batch_size`: 128
275
+ - `prediction_loss_only`: True
276
+ - `eval_on_start`: False
277
+ - `eval_do_concat_batches`: True
278
+ - `eval_use_gather_object`: False
279
+ - `eval_accumulation_steps`: None
280
+ - `include_for_metrics`: []
281
+ - `batch_eval_metrics`: False
282
+ - `save_only_model`: False
283
+ - `save_on_each_node`: False
284
+ - `enable_jit_checkpoint`: False
285
+ - `push_to_hub`: False
286
+ - `hub_private_repo`: None
287
+ - `hub_model_id`: None
288
+ - `hub_strategy`: every_save
289
+ - `hub_always_push`: False
290
+ - `hub_revision`: None
291
+ - `load_best_model_at_end`: False
292
+ - `ignore_data_skip`: False
293
+ - `restore_callback_states_from_checkpoint`: False
294
+ - `full_determinism`: False
295
+ - `seed`: 42
296
+ - `data_seed`: None
297
+ - `use_cpu`: False
298
+ - `accelerator_config`: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
299
+ - `parallelism_config`: None
300
+ - `dataloader_drop_last`: False
301
+ - `dataloader_num_workers`: 0
302
+ - `dataloader_pin_memory`: False
303
+ - `dataloader_persistent_workers`: False
304
+ - `dataloader_prefetch_factor`: None
305
+ - `dataloader_multiprocessing_context`: None
306
+ - `dataloader_in_order`: True
307
+ - `remove_unused_columns`: True
308
+ - `label_names`: None
309
+ - `train_sampling_strategy`: random
310
+ - `length_column_name`: length
311
+ - `ddp_find_unused_parameters`: None
312
+ - `ddp_bucket_cap_mb`: None
313
+ - `ddp_broadcast_buffers`: False
314
+ - `ddp_static_graph`: None
315
+ - `ddp_backend`: None
316
+ - `ddp_timeout`: 1800
317
+ - `fsdp`: None
318
+ - `fsdp_config`: None
319
+ - `deepspeed`: None
320
+ - `debug`: []
321
+ - `skip_memory_metrics`: True
322
+ - `do_predict`: False
323
+ - `resume_from_checkpoint`: None
324
+ - `local_rank`: -1
325
+ - `prompts`: None
326
+ - `batch_sampler`: batch_sampler
327
+ - `multi_dataset_batch_sampler`: proportional
328
+ - `router_mapping`: {}
329
+ - `learning_rate_mapping`: {}
330
+ - `warmup_ratio`: None
331
+
332
+ </details>
333
+
334
+ ### Training Logs
335
+ | Epoch | Step | Training Loss | Validation Loss |
336
+ |:------:|:----:|:-------------:|:---------------:|
337
+ | 0.1156 | 100 | - | 0.2757 |
338
+ | 0.2312 | 200 | - | 0.2636 |
339
+ | 0.3468 | 300 | - | 0.2548 |
340
+ | 0.4624 | 400 | - | 0.2484 |
341
+ | 0.5780 | 500 | 0.2929 | 0.2452 |
342
+ | 0.6936 | 600 | - | 0.2387 |
343
+ | 0.8092 | 700 | - | 0.2291 |
344
+ | 0.9249 | 800 | - | 0.2355 |
345
+ | 1.0405 | 900 | - | 0.2288 |
346
+ | 1.1561 | 1000 | 0.2378 | 0.2275 |
347
+ | 1.2717 | 1100 | - | 0.2263 |
348
+ | 1.3873 | 1200 | - | 0.2282 |
349
+ | 1.5029 | 1300 | - | 0.2228 |
350
+ | 1.6185 | 1400 | - | 0.2199 |
351
+ | 1.7341 | 1500 | 0.1797 | 0.2235 |
352
+ | 1.8497 | 1600 | - | 0.2181 |
353
+ | 1.9653 | 1700 | - | 0.2214 |
354
+ | 2.0809 | 1800 | - | 0.2164 |
355
+ | 2.1965 | 1900 | - | 0.2150 |
356
+ | 2.3121 | 2000 | 0.1466 | 0.2133 |
357
+ | 2.4277 | 2100 | - | 0.2122 |
358
+ | 2.5434 | 2200 | - | 0.2117 |
359
+ | 2.6590 | 2300 | - | 0.2118 |
360
+ | 2.7746 | 2400 | - | 0.2113 |
361
+ | 2.8902 | 2500 | 0.1352 | 0.2097 |
362
+ | 3.0058 | 2600 | - | 0.2075 |
363
+ | 3.1214 | 2700 | - | 0.2110 |
364
+ | 3.2370 | 2800 | - | 0.2114 |
365
+ | 3.3526 | 2900 | - | 0.2170 |
366
+ | 3.4682 | 3000 | 0.1012 | 0.2143 |
367
+ | 3.5838 | 3100 | - | 0.2110 |
368
+
369
+
370
+ ### Training Time
371
+ - **Training**: 20.9 minutes
372
+ - **Evaluation**: 13.3 minutes
373
+ - **Total**: 34.2 minutes
374
+
375
+ ### Framework Versions
376
+ - Python: 3.14.7
377
+ - Sentence Transformers: 6.2.0
378
+ - Transformers: 5.22.0
379
+ - PyTorch: 2.15.0
380
+ - Accelerate: 1.15.0
381
+ - Datasets: 5.1.0
382
+ - Tokenizers: 0.23.3
383
+
384
+ ## Additional Resources
385
+
386
+ - [Training and Finetuning Embedding Models with Sentence Transformers](https://huggingface.co/blog/train-sentence-transformers): the end-to-end guide for training or finetuning Sentence Transformer models.
387
+ - [Introduction to Matryoshka Embedding Models](https://huggingface.co/blog/matryoshka): variable-size embeddings that can be truncated with minimal quality loss.
388
+ - [Binary and Scalar Embedding Quantization for Significantly Faster & Cheaper Retrieval](https://huggingface.co/blog/embedding-quantization): post-training compression of embedding vectors.
389
+ - [Multimodal Embedding & Reranker Models with Sentence Transformers](https://huggingface.co/blog/multimodal-sentence-transformers): use text, image, audio, and video models through the same API.
390
+ - [Training and Finetuning Multimodal Embedding & Reranker Models with Sentence Transformers](https://huggingface.co/blog/train-multimodal-sentence-transformers): train multimodal embedding models, with a Visual Document Retrieval walkthrough.
391
+
392
+ ## Citation
393
+
394
+ ### BibTeX
395
+
396
+ #### Sentence Transformers
397
+ ```bibtex
398
+ @inproceedings{reimers-2019-sentence-bert,
399
+ title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
400
+ author = "Reimers, Nils and Gurevych, Iryna",
401
+ booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
402
+ month = "11",
403
+ year = "2019",
404
+ publisher = "Association for Computational Linguistics",
405
+ url = "https://arxiv.org/abs/1908.10084",
406
+ }
407
+ ```
408
+
409
+ #### MultipleNegativesRankingLoss
410
+ ```bibtex
411
+ @misc{oord2019representationlearningcontrastivepredictive,
412
+ title={Representation Learning with Contrastive Predictive Coding},
413
+ author={Aaron van den Oord and Yazhe Li and Oriol Vinyals},
414
+ year={2019},
415
+ eprint={1807.03748},
416
+ archivePrefix={arXiv},
417
+ primaryClass={cs.LG},
418
+ url={https://arxiv.org/abs/1807.03748},
419
+ }
420
+ ```
421
+
422
+ <!--
423
+ ## Glossary
424
+
425
+ *Clearly define terms in order to be accessible across audiences.*
426
+ -->
427
+
428
+ <!--
429
+ ## Model Card Authors
430
+
431
+ *Lists the people who create the model card, providing recognition and accountability for the detailed work that goes into its construction.*
432
+ -->
433
+
434
+ <!--
435
+ ## Model Card Contact
436
+
437
+ *Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors.*
438
+ -->
checkpoint-3100/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_cross_attention": false,
3
+ "architectures": [
4
+ "BertModel"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": null,
8
+ "classifier_dropout": null,
9
+ "dtype": "float32",
10
+ "eos_token_id": null,
11
+ "gradient_checkpointing": false,
12
+ "hidden_act": "gelu",
13
+ "hidden_dropout_prob": 0.1,
14
+ "hidden_size": 384,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 1536,
17
+ "is_decoder": false,
18
+ "layer_norm_eps": 1e-12,
19
+ "max_position_embeddings": 512,
20
+ "model_type": "bert",
21
+ "num_attention_heads": 12,
22
+ "num_hidden_layers": 6,
23
+ "pad_token_id": 0,
24
+ "position_embedding_type": "absolute",
25
+ "tie_word_embeddings": true,
26
+ "transformers_version": "5.22.0",
27
+ "type_vocab_size": 2,
28
+ "use_cache": false,
29
+ "vocab_size": 30522
30
+ }
checkpoint-3100/config_sentence_transformers.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "__version__": {
3
+ "pytorch": "2.15.0",
4
+ "sentence_transformers": "6.2.0",
5
+ "transformers": "5.22.0"
6
+ },
7
+ "default_prompt_name": null,
8
+ "model_type": "SentenceTransformer",
9
+ "prompts": {
10
+ "document": "",
11
+ "query": ""
12
+ },
13
+ "similarity_fn_name": "cosine"
14
+ }
checkpoint-3100/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0ac6d2340719a9c19c68dcbf12bc352d2048cd2c8396bf42b2cf02b1024541d
3
+ size 90864192
checkpoint-3100/modules.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "idx": 0,
4
+ "name": "0",
5
+ "path": "",
6
+ "type": "sentence_transformers.base.modules.transformer.Transformer"
7
+ },
8
+ {
9
+ "idx": 1,
10
+ "name": "1",
11
+ "path": "1_Pooling",
12
+ "type": "sentence_transformers.sentence_transformer.modules.pooling.Pooling"
13
+ },
14
+ {
15
+ "idx": 2,
16
+ "name": "2",
17
+ "path": "2_Normalize",
18
+ "type": "sentence_transformers.base.modules.normalize.Normalize"
19
+ }
20
+ ]
checkpoint-3100/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2629417154b8c99a8cc4e3de71ca3afcea57ca36eac16b9167edc213a9c08c1c
3
+ size 180605387
checkpoint-3100/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a13b429148ee64e14f1d390f959b55abd863063f6a2d81e199a4d26ccde313e
3
+ size 14391
checkpoint-3100/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c00e65ec016f7112554534e3cf74ed675906d973e79501cfa84efec4881dffe2
3
+ size 1465
checkpoint-3100/sentence_bert_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "transformer_task": "feature-extraction",
3
+ "modality_config": {
4
+ "text": {
5
+ "method": "forward",
6
+ "method_output_name": "last_hidden_state"
7
+ }
8
+ },
9
+ "module_output_name": "token_embeddings"
10
+ }
checkpoint-3100/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-3100/tokenizer_config.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "cls_token": "[CLS]",
4
+ "do_basic_tokenize": true,
5
+ "do_lower_case": true,
6
+ "is_local": false,
7
+ "local_files_only": false,
8
+ "mask_token": "[MASK]",
9
+ "max_length": 128,
10
+ "model_max_length": 256,
11
+ "never_split": null,
12
+ "pad_to_multiple_of": null,
13
+ "pad_token": "[PAD]",
14
+ "pad_token_type_id": 0,
15
+ "padding_side": "right",
16
+ "sep_token": "[SEP]",
17
+ "stride": 0,
18
+ "strip_accents": null,
19
+ "tokenize_chinese_chars": true,
20
+ "tokenizer_class": "BertTokenizer",
21
+ "truncation_side": "right",
22
+ "truncation_strategy": "longest_first",
23
+ "unk_token": "[UNK]"
24
+ }
checkpoint-3100/trainer_state.json ADDED
@@ -0,0 +1,333 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 2600,
3
+ "best_metric": 0.2075337916612625,
4
+ "best_model_checkpoint": "ft/sentence-transformers/all-MiniLM-L6-v2/checkpoint-2600",
5
+ "epoch": 3.583815028901734,
6
+ "eval_steps": 100,
7
+ "global_step": 3100,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.11560693641618497,
14
+ "eval_loss": 0.2756878137588501,
15
+ "eval_runtime": 19.195,
16
+ "eval_samples_per_second": 1440.58,
17
+ "eval_steps_per_second": 11.305,
18
+ "step": 100
19
+ },
20
+ {
21
+ "epoch": 0.23121387283236994,
22
+ "eval_loss": 0.2635912597179413,
23
+ "eval_runtime": 19.1881,
24
+ "eval_samples_per_second": 1441.105,
25
+ "eval_steps_per_second": 11.309,
26
+ "step": 200
27
+ },
28
+ {
29
+ "epoch": 0.3468208092485549,
30
+ "eval_loss": 0.25484976172447205,
31
+ "eval_runtime": 20.4586,
32
+ "eval_samples_per_second": 1351.608,
33
+ "eval_steps_per_second": 10.607,
34
+ "step": 300
35
+ },
36
+ {
37
+ "epoch": 0.4624277456647399,
38
+ "eval_loss": 0.24843913316726685,
39
+ "eval_runtime": 24.2538,
40
+ "eval_samples_per_second": 1140.109,
41
+ "eval_steps_per_second": 8.947,
42
+ "step": 400
43
+ },
44
+ {
45
+ "epoch": 0.5780346820809249,
46
+ "grad_norm": 3.031534194946289,
47
+ "learning_rate": 9.884624277456648e-05,
48
+ "loss": 0.2928614196777344,
49
+ "step": 500
50
+ },
51
+ {
52
+ "epoch": 0.5780346820809249,
53
+ "eval_loss": 0.24522048234939575,
54
+ "eval_runtime": 24.3615,
55
+ "eval_samples_per_second": 1135.067,
56
+ "eval_steps_per_second": 8.907,
57
+ "step": 500
58
+ },
59
+ {
60
+ "epoch": 0.6936416184971098,
61
+ "eval_loss": 0.23871970176696777,
62
+ "eval_runtime": 23.7379,
63
+ "eval_samples_per_second": 1164.886,
64
+ "eval_steps_per_second": 9.141,
65
+ "step": 600
66
+ },
67
+ {
68
+ "epoch": 0.8092485549132948,
69
+ "eval_loss": 0.22912940382957458,
70
+ "eval_runtime": 23.4904,
71
+ "eval_samples_per_second": 1177.162,
72
+ "eval_steps_per_second": 9.238,
73
+ "step": 700
74
+ },
75
+ {
76
+ "epoch": 0.9248554913294798,
77
+ "eval_loss": 0.23552891612052917,
78
+ "eval_runtime": 23.3878,
79
+ "eval_samples_per_second": 1182.328,
80
+ "eval_steps_per_second": 9.278,
81
+ "step": 800
82
+ },
83
+ {
84
+ "epoch": 1.0404624277456647,
85
+ "eval_loss": 0.22875414788722992,
86
+ "eval_runtime": 24.2858,
87
+ "eval_samples_per_second": 1138.608,
88
+ "eval_steps_per_second": 8.935,
89
+ "step": 900
90
+ },
91
+ {
92
+ "epoch": 1.1560693641618498,
93
+ "grad_norm": 2.5727860927581787,
94
+ "learning_rate": 9.769017341040463e-05,
95
+ "loss": 0.2378170166015625,
96
+ "step": 1000
97
+ },
98
+ {
99
+ "epoch": 1.1560693641618498,
100
+ "eval_loss": 0.22745409607887268,
101
+ "eval_runtime": 24.6958,
102
+ "eval_samples_per_second": 1119.705,
103
+ "eval_steps_per_second": 8.787,
104
+ "step": 1000
105
+ },
106
+ {
107
+ "epoch": 1.2716763005780347,
108
+ "eval_loss": 0.22631044685840607,
109
+ "eval_runtime": 25.1507,
110
+ "eval_samples_per_second": 1099.453,
111
+ "eval_steps_per_second": 8.628,
112
+ "step": 1100
113
+ },
114
+ {
115
+ "epoch": 1.3872832369942196,
116
+ "eval_loss": 0.22824475169181824,
117
+ "eval_runtime": 25.1355,
118
+ "eval_samples_per_second": 1100.116,
119
+ "eval_steps_per_second": 8.633,
120
+ "step": 1200
121
+ },
122
+ {
123
+ "epoch": 1.5028901734104045,
124
+ "eval_loss": 0.22276242077350616,
125
+ "eval_runtime": 25.2896,
126
+ "eval_samples_per_second": 1093.414,
127
+ "eval_steps_per_second": 8.581,
128
+ "step": 1300
129
+ },
130
+ {
131
+ "epoch": 1.6184971098265897,
132
+ "eval_loss": 0.21992968022823334,
133
+ "eval_runtime": 25.7239,
134
+ "eval_samples_per_second": 1074.952,
135
+ "eval_steps_per_second": 8.436,
136
+ "step": 1400
137
+ },
138
+ {
139
+ "epoch": 1.7341040462427746,
140
+ "grad_norm": 2.2508647441864014,
141
+ "learning_rate": 9.653410404624278e-05,
142
+ "loss": 0.1796602783203125,
143
+ "step": 1500
144
+ },
145
+ {
146
+ "epoch": 1.7341040462427746,
147
+ "eval_loss": 0.2235146462917328,
148
+ "eval_runtime": 27.2082,
149
+ "eval_samples_per_second": 1016.311,
150
+ "eval_steps_per_second": 7.976,
151
+ "step": 1500
152
+ },
153
+ {
154
+ "epoch": 1.8497109826589595,
155
+ "eval_loss": 0.21805228292942047,
156
+ "eval_runtime": 26.5895,
157
+ "eval_samples_per_second": 1039.959,
158
+ "eval_steps_per_second": 8.161,
159
+ "step": 1600
160
+ },
161
+ {
162
+ "epoch": 1.9653179190751446,
163
+ "eval_loss": 0.22142848372459412,
164
+ "eval_runtime": 26.6159,
165
+ "eval_samples_per_second": 1038.929,
166
+ "eval_steps_per_second": 8.153,
167
+ "step": 1700
168
+ },
169
+ {
170
+ "epoch": 2.0809248554913293,
171
+ "eval_loss": 0.21644671261310577,
172
+ "eval_runtime": 26.2398,
173
+ "eval_samples_per_second": 1053.819,
174
+ "eval_steps_per_second": 8.27,
175
+ "step": 1800
176
+ },
177
+ {
178
+ "epoch": 2.1965317919075145,
179
+ "eval_loss": 0.2150029093027115,
180
+ "eval_runtime": 25.8402,
181
+ "eval_samples_per_second": 1070.114,
182
+ "eval_steps_per_second": 8.398,
183
+ "step": 1900
184
+ },
185
+ {
186
+ "epoch": 2.3121387283236996,
187
+ "grad_norm": 1.9824634790420532,
188
+ "learning_rate": 9.537803468208093e-05,
189
+ "loss": 0.14658987426757814,
190
+ "step": 2000
191
+ },
192
+ {
193
+ "epoch": 2.3121387283236996,
194
+ "eval_loss": 0.213293194770813,
195
+ "eval_runtime": 25.5778,
196
+ "eval_samples_per_second": 1081.094,
197
+ "eval_steps_per_second": 8.484,
198
+ "step": 2000
199
+ },
200
+ {
201
+ "epoch": 2.4277456647398843,
202
+ "eval_loss": 0.21219080686569214,
203
+ "eval_runtime": 25.765,
204
+ "eval_samples_per_second": 1073.239,
205
+ "eval_steps_per_second": 8.422,
206
+ "step": 2100
207
+ },
208
+ {
209
+ "epoch": 2.5433526011560694,
210
+ "eval_loss": 0.2117120325565338,
211
+ "eval_runtime": 26.1201,
212
+ "eval_samples_per_second": 1058.648,
213
+ "eval_steps_per_second": 8.308,
214
+ "step": 2200
215
+ },
216
+ {
217
+ "epoch": 2.6589595375722546,
218
+ "eval_loss": 0.21178804337978363,
219
+ "eval_runtime": 26.1939,
220
+ "eval_samples_per_second": 1055.664,
221
+ "eval_steps_per_second": 8.284,
222
+ "step": 2300
223
+ },
224
+ {
225
+ "epoch": 2.7745664739884393,
226
+ "eval_loss": 0.2113269418478012,
227
+ "eval_runtime": 26.5141,
228
+ "eval_samples_per_second": 1042.916,
229
+ "eval_steps_per_second": 8.184,
230
+ "step": 2400
231
+ },
232
+ {
233
+ "epoch": 2.8901734104046244,
234
+ "grad_norm": 1.6065114736557007,
235
+ "learning_rate": 9.422196531791908e-05,
236
+ "loss": 0.13519590759277345,
237
+ "step": 2500
238
+ },
239
+ {
240
+ "epoch": 2.8901734104046244,
241
+ "eval_loss": 0.20970419049263,
242
+ "eval_runtime": 26.2255,
243
+ "eval_samples_per_second": 1054.393,
244
+ "eval_steps_per_second": 8.274,
245
+ "step": 2500
246
+ },
247
+ {
248
+ "epoch": 3.005780346820809,
249
+ "eval_loss": 0.2075337916612625,
250
+ "eval_runtime": 26.4595,
251
+ "eval_samples_per_second": 1045.07,
252
+ "eval_steps_per_second": 8.201,
253
+ "step": 2600
254
+ },
255
+ {
256
+ "epoch": 3.121387283236994,
257
+ "eval_loss": 0.21102772653102875,
258
+ "eval_runtime": 26.5553,
259
+ "eval_samples_per_second": 1041.298,
260
+ "eval_steps_per_second": 8.172,
261
+ "step": 2700
262
+ },
263
+ {
264
+ "epoch": 3.2369942196531793,
265
+ "eval_loss": 0.21144026517868042,
266
+ "eval_runtime": 28.0952,
267
+ "eval_samples_per_second": 984.224,
268
+ "eval_steps_per_second": 7.724,
269
+ "step": 2800
270
+ },
271
+ {
272
+ "epoch": 3.352601156069364,
273
+ "eval_loss": 0.21704500913619995,
274
+ "eval_runtime": 47.621,
275
+ "eval_samples_per_second": 580.669,
276
+ "eval_steps_per_second": 4.557,
277
+ "step": 2900
278
+ },
279
+ {
280
+ "epoch": 3.468208092485549,
281
+ "grad_norm": 1.3753323554992676,
282
+ "learning_rate": 9.306589595375723e-05,
283
+ "loss": 0.10123191833496094,
284
+ "step": 3000
285
+ },
286
+ {
287
+ "epoch": 3.468208092485549,
288
+ "eval_loss": 0.21425534784793854,
289
+ "eval_runtime": 26.2365,
290
+ "eval_samples_per_second": 1053.951,
291
+ "eval_steps_per_second": 8.271,
292
+ "step": 3000
293
+ },
294
+ {
295
+ "epoch": 3.583815028901734,
296
+ "eval_loss": 0.2110053449869156,
297
+ "eval_runtime": 26.3915,
298
+ "eval_samples_per_second": 1047.76,
299
+ "eval_steps_per_second": 8.222,
300
+ "step": 3100
301
+ }
302
+ ],
303
+ "logging_steps": 500,
304
+ "max_steps": 43250,
305
+ "num_input_tokens_seen": 0,
306
+ "num_train_epochs": 50,
307
+ "save_steps": 100,
308
+ "stateful_callbacks": {
309
+ "EarlyStoppingCallback": {
310
+ "args": {
311
+ "early_stopping_patience": 5,
312
+ "early_stopping_threshold": 0.0
313
+ },
314
+ "attributes": {
315
+ "early_stopping_patience_counter": 5
316
+ }
317
+ },
318
+ "TrainerControl": {
319
+ "args": {
320
+ "should_epoch_stop": false,
321
+ "should_evaluate": false,
322
+ "should_log": false,
323
+ "should_save": true,
324
+ "should_training_stop": true
325
+ },
326
+ "attributes": {}
327
+ }
328
+ },
329
+ "total_flos": 0.0,
330
+ "train_batch_size": 128,
331
+ "trial_name": null,
332
+ "trial_params": null
333
+ }
checkpoint-3100/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbaa10d98d1cdd9a980547ffbef8974da829ab33a5dd461feba1e6741bdeb5fc
3
+ size 5585
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_cross_attention": false,
3
+ "architectures": [
4
+ "BertModel"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": null,
8
+ "classifier_dropout": null,
9
+ "dtype": "float32",
10
+ "eos_token_id": null,
11
+ "gradient_checkpointing": false,
12
+ "hidden_act": "gelu",
13
+ "hidden_dropout_prob": 0.1,
14
+ "hidden_size": 384,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 1536,
17
+ "is_decoder": false,
18
+ "layer_norm_eps": 1e-12,
19
+ "max_position_embeddings": 512,
20
+ "model_type": "bert",
21
+ "num_attention_heads": 12,
22
+ "num_hidden_layers": 6,
23
+ "pad_token_id": 0,
24
+ "position_embedding_type": "absolute",
25
+ "tie_word_embeddings": true,
26
+ "transformers_version": "5.22.0",
27
+ "type_vocab_size": 2,
28
+ "use_cache": false,
29
+ "vocab_size": 30522
30
+ }
config_sentence_transformers.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "__version__": {
3
+ "pytorch": "2.15.0",
4
+ "sentence_transformers": "6.2.0",
5
+ "transformers": "5.22.0"
6
+ },
7
+ "default_prompt_name": null,
8
+ "model_type": "SentenceTransformer",
9
+ "prompts": {
10
+ "document": "",
11
+ "query": ""
12
+ },
13
+ "similarity_fn_name": "cosine"
14
+ }