Hugging Face's logo

skumar1998
/
result

Transformers
TensorBoard
Safetensors
Generated from Trainer
unsloth
sft
trl
Model card Files
xet
 Metrics Community
skumar1998 commited on
Commit
96c71d4
·
verified ·
1 Parent(s): f96591a

Training in progress, step 158

Browse files
Files changed (7) hide show
  1. adapter_config.json +3 -3
  2. adapter_model.safetensors +2 -2
  3. file.log +42 -43
  4. ground_truth_data.json +0 -0
  5. gt_json.json +0 -0
  6. runs/Aug23_12-41-40_3c59ea7c2175/events.out.tfevents.1787488900.3c59ea7c2175.10055.0 +3 -0
  7. training_args.bin +1 -1
adapter_config.json CHANGED
@@ -20,7 +20,7 @@
20
  "layers_pattern": null,
21
  "layers_to_transform": null,
22
  "loftq_config": {},
23
- "lora_alpha": 256,
24
  "lora_bias": false,
25
  "lora_dropout": 0.05,
26
  "lora_ga_config": null,
@@ -31,10 +31,10 @@
31
  "peft_type": "LORA",
32
  "peft_version": "0.20.0",
33
  "qalora_group_size": 16,
34
- "r": 128,
35
  "rank_pattern": {},
36
  "revision": null,
37
- "target_modules": "(?:.*?(?:vision|image|visual|patch|language|text).*?(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer).*?(?:qkv|proj|linear_fc1|linear_fc2|out_proj|in_proj_qkv|in_proj_z|in_proj_b|in_proj_a|gate_proj|up_proj|down_proj|q_proj|k_proj|v_proj|o_proj))|(?:\\bmodel\\.layers\\.[\\d]{1,}\\.(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer)\\.(?:(?:qkv|proj|linear_fc1|linear_fc2|out_proj|in_proj_qkv|in_proj_z|in_proj_b|in_proj_a|gate_proj|up_proj|down_proj|q_proj|k_proj|v_proj|o_proj)))",
38
  "target_parameters": null,
39
  "task_type": "CAUSAL_LM",
40
  "trainable_token_indices": null,
 
20
  "layers_pattern": null,
21
  "layers_to_transform": null,
22
  "loftq_config": {},
23
+ "lora_alpha": 512,
24
  "lora_bias": false,
25
  "lora_dropout": 0.05,
26
  "lora_ga_config": null,
 
31
  "peft_type": "LORA",
32
  "peft_version": "0.20.0",
33
  "qalora_group_size": 16,
34
+ "r": 256,
35
  "rank_pattern": {},
36
  "revision": null,
37
+ "target_modules": "(?:.*?(?:language|text).*?(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer).*?(?:qkv|proj|linear_fc1|linear_fc2|out_proj|in_proj_qkv|in_proj_z|in_proj_b|in_proj_a|gate_proj|up_proj|down_proj|q_proj|k_proj|v_proj|o_proj))|(?:\\bmodel\\.layers\\.[\\d]{1,}\\.(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer)\\.(?:(?:qkv|proj|linear_fc1|linear_fc2|out_proj|in_proj_qkv|in_proj_z|in_proj_b|in_proj_a|gate_proj|up_proj|down_proj|q_proj|k_proj|v_proj|o_proj)))",
38
  "target_parameters": null,
39
  "task_type": "CAUSAL_LM",
40
  "trainable_token_indices": null,
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:bc9b3b0f5cd6389a6a41a365e3c5ee0a4d1781744acca1b53d2c38d3efd12964
3
- size 3981873664
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:afe495b7614fb68baf0dccde6d0052b02f6d68b484be37c3838e18f16f0ffe6a
3
+ size 7470734200
file.log CHANGED
@@ -1,44 +1,43 @@
1
- 2026-08-18 20:28:54.494 | INFO | app.tasks.entity.vlm.dataset:create:708 - Using pre-split dataset mode
2
- 2026-08-18 20:28:54.514 | INFO | app.tasks.entity.vlm.dataset:_download_current_splits:1623 - Downloading train dataset from s3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/train.json
3
- 2026-08-18 20:28:55.201 | INFO | app.tasks.entity.vlm.dataset:_download_split_documents:1480 - Processing %s %s/%s with %d documents
4
- 2026-08-18 20:29:32.249 | INFO | app.tasks.entity.vlm.dataset:_download_current_splits:1623 - Downloading val dataset from s3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/eval.json
5
- 2026-08-18 20:29:32.618 | INFO | app.tasks.entity.vlm.dataset:_download_split_documents:1480 - Processing %s %s/%s with %d documents
6
- 2026-08-18 20:29:37.218 | INFO | app.tasks.entity.vlm.dataset:_download_current_splits:1623 - Downloading test dataset from s3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/test.json
7
- 2026-08-18 20:29:37.784 | INFO | app.tasks.entity.vlm.dataset:_download_split_documents:1480 - Processing %s %s/%s with %d documents
8
- 2026-08-18 20:29:48.290 | INFO | app.tasks.entity.vlm.dataset:_download_miscellaneous:1670 - Downloading miscellaneous data from s3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/miscellaneous.json
9
- 2026-08-18 20:29:48.862 | INFO | app.tasks.entity.vlm.dataset:_download_previous_datasets:1579 - Found 0 previous dataset(s) to consolidate
10
- 2026-08-18 20:29:48.863 | INFO | app.tasks.entity.vlm.dataset:download_presplit_datasets:1721 - Successfully downloaded all datasets to /home/ubuntu/dataset
11
- 2026-08-18 20:30:09.684 | INFO | app.tasks.entity.vlm.dataset:remove_null_files:632 - Removed 2 files with null values
12
- 2026-08-18 20:30:11.810 | INFO | app.tasks.entity.vlm.reports:get_distribution:1520 - Original number of ground truth files 6058
13
- 2026-08-18 20:30:18.067 | INFO | app.tasks.entity.vlm.reports:get_distribution:1535 - After removing empty files 6058
14
- 2026-08-18 20:30:18.310 | INFO | app.tasks.entity.vlm.reports:get_entity_distribution:1635 - Creating entity distribution for 6058 files
15
- 2026-08-18 20:30:23.326 | DEBUG | app.commonmlops:get_oauth_token:834 - OAuth token request status: 200
16
- 2026-08-18 20:30:23.589 | DEBUG | app.commonmlops:get_training_job_queue:891 - Get training job queue status code: 200, api_url: https://aitest.api.dexit.tech/api/v1/mlops/trainingjobqueue/06a665f8-ebd0-7481-8000-83304d751ace
17
- 2026-08-18 20:30:23.811 | DEBUG | app.commonmlops:get_oauth_token:834 - OAuth token request status: 200
18
- 2026-08-18 20:30:23.812 | DEBUG | app.commonmlops:update_training_job_queue:937 - Updating training job queue with merged payload: {'id': '06a665f8-ebd0-7481-8000-83304d751ace', 'modeltype': 'entity', 'modelname': 'vlm_qwen3_6', 'jobtype': 'feedback', 'basemodelversionid': '019f0d56-3844-7c0a-8fab-4aa6a6d1f12e', 'datasetid': '06a665f5-06bf-7f15-8000-acbbe709e051', 'status': 'triggered', 'createddate': '2026-07-26T19:27:10.738422Z', 'createdby': '00000000-0000-0000-0000-000000000000', 'startdate': '2026-08-18T20:28:23.455801Z', 'enddate': None, 'accuracyreport': {'overall': {'recall': 0.9489486112672889, 'f1_score': 0.9511143670226255, 'precision': 0.953290031064671}, 'patient': {'recall': 0.9570468593654881, 'accuracy': 0.965850180851059, 'f1_score': 0.9578867442511313, 'precision': 0.9588553005522813}, 'facility': {'recall': 0.9681215596546779, 'accuracy': 0.9468811825459762, 'f1_score': 0.9706030656585861, 'precision': 0.9731011770220469}, 'patient-dob': {'recall': 0.9812606473594548, 'accuracy': 0.9683042789223455, 'f1_score': 0.9829351535836178, 'precision': 0.9846153846153847}, 'patient-sex': {'recall': 0.9982847341337907, 'accuracy': 0.9967871485943776, 'f1_score': 0.9982847341337907, 'precision': 0.9982847341337907}, 'patient-zip': {'recall': 0.9623762376237623, 'accuracy': 0.9839486356340289, 'f1_score': 0.9798387096774194, 'precision': 0.997946611909651}, 'patient.dob': {'recall': 0.9821124361158433, 'accuracy': 0.9706582077716098, 'f1_score': 0.9842082799829279, 'precision': 0.9863130881094953}, 'patient.sex': {'recall': 0.9991423670668954, 'accuracy': 0.9983935742971888, 'f1_score': 0.9991423670668954, 'precision': 0.9991423670668954}, 'patient.zip': {'recall': 0.9663366336633663, 'accuracy': 0.9847512038523274, 'f1_score': 0.9809045226130653, 'precision': 0.9959183673469387}, 'sample-code': {'recall': 0.9737609329446064, 'accuracy': 0.9545454545454546, 'f1_score': 0.9732880038853812, 'precision': 0.9728155339805825}, 'facility-zip': {'recall': 0.9796782387806944, 'accuracy': 0.9728434504792333, 'f1_score': 0.985519591141397, 'precision': 0.9914310197086547}, 'facility.zip': {'recall': 0.9847586790855207, 'accuracy': 0.9808, 'f1_score': 0.9897872340425531, 'precision': 0.9948674080410608}, 'patient-city': {'recall': 0.9756592292089249, 'accuracy': 0.9864, 'f1_score': 0.982635342185904, 'precision': 0.9897119341563786}, 'patient.city': {'recall': 0.9817444219066938, 'accuracy': 0.988, 'f1_score': 0.9847405900305187, 'precision': 0.9877551020408163}, 'phone-number': {'recall': 0.9784172661870504, 'accuracy': 0.9847756410256411, 'f1_score': 0.9662522202486679, 'precision': 0.9543859649122807}, 'provider-npi': {'recall': 1.0, 'accuracy': 0.9991546914623838, 'f1_score': 0.9995147986414362, 'precision': 0.9990300678952473}, 'facility-city': {'recall': 0.9931682322801024, 'accuracy': 0.9903691813804173, 'f1_score': 0.9948674080410608, 'precision': 0.9965724078834619}, 'facility-name': {'recall': 0.9692570452604612, 'accuracy': 0.9460093896713615, 'f1_score': 0.9705002137665669, 'precision': 0.9717465753424658}, 'facility.city': {'recall': 0.9965841161400513, 'accuracy': 0.9967871485943776, 'f1_score': 0.998289136013687, 'precision': 1.0}, 'facility.name': {'recall': 0.9624252775405636, 'accuracy': 0.9339035769828927, 'f1_score': 0.9636596836254809, 'precision': 0.9648972602739726}, 'patient-state': {'recall': 0.7729007633587787, 'accuracy': 0.8273062730627306, 'f1_score': 0.7758620689655172, 'precision': 0.7788461538461539}, 'patient.state': {'recall': 0.8530534351145038, 'accuracy': 0.8843226788432268, 'f1_score': 0.8546845124282982, 'precision': 0.8563218390804598}, 'facility-state': {'recall': 0.8662691652470187, 'accuracy': 0.7794540229885057, 'f1_score': 0.8688594617684751, 'precision': 0.87146529562982}, 'facility.state': {'recall': 0.9335604770017035, 'accuracy': 0.8867781155015197, 'f1_score': 0.936351986330628, 'precision': 0.9391602399314481}, 'collection-date': {'recall': 0.8938053097345132, 'accuracy': 0.9175597691673537, 'f1_score': 0.9099099099099099, 'precision': 0.926605504587156}, 'patient-address': {'recall': 0.8693877551020408, 'accuracy': 0.9042145593869731, 'f1_score': 0.872057318321392, 'precision': 0.8747433264887063}, 'patient.address': {'recall': 0.9, 'accuracy': 0.9255813953488372, 'f1_score': 0.901840490797546, 'precision': 0.9036885245901639}, 'accession-number': {'recall': 0.9967948717948718, 'accuracy': 0.9991546914623838, 'f1_score': 0.9983948635634029, 'precision': 1.0}, 'facility-address': {'recall': 0.9513236549957301, 'accuracy': 0.9173106646058733, 'f1_score': 0.9541755888650965, 'precision': 0.9570446735395189}, 'facility.address': {'recall': 0.9632792485055508, 'accuracy': 0.9361370716510904, 'f1_score': 0.9649272882805817, 'precision': 0.9665809768637532}, 'patient-last-name': {'recall': 0.9879518072289156, 'accuracy': 0.9785202863961814, 'f1_score': 0.9891435464414957, 'precision': 0.9903381642512077}, 'patient.last-name': {'recall': 0.9879518072289156, 'accuracy': 0.9785202863961814, 'f1_score': 0.9891435464414957, 'precision': 0.9903381642512077}, 'patient-first-name': {'recall': 0.9927652733118971, 'accuracy': 0.9872102318145484, 'f1_score': 0.9935639581657282, 'precision': 0.9943639291465378}, 'patient.first-name': {'recall': 0.9911575562700965, 'accuracy': 0.9848242811501597, 'f1_score': 0.9923541247484909, 'precision': 0.9935535858178888}, 'patient-middle-name': {'recall': 0.9182692307692307, 'accuracy': 0.9641434262948207, 'f1_score': 0.8946135831381734, 'precision': 0.8721461187214612}, 'patient.middle-name': {'recall': 0.9519230769230769, 'accuracy': 0.9776, 'f1_score': 0.9339622641509433, 'precision': 0.9166666666666666}, 'other-insurance-name-1': {'recall': 0.7476190476190476, 'accuracy': 0.8555900621118012, 'f1_score': 0.7714987714987716, 'precision': 0.7969543147208121}, 'other-insurance-name-2': {'recall': 1.0, 'accuracy': 0.9975903614457832, 'f1_score': 0.4, 'precision': 0.25}, 'social-security-number': {'recall': 0.9210526315789473, 'accuracy': 0.9847634322373697, 'f1_score': 0.7865168539325842, 'precision': 0.6862745098039216}, 'other-insurance-number-1': {'recall': 0.8333333333333334, 'accuracy': 0.9074221867517956, 'f1_score': 0.857843137254902, 'precision': 0.8838383838383839}, 'other-insurance-number-2': {'recall': 1.0, 'accuracy': 0.9975903614457832, 'f1_score': 0.4, 'precision': 0.25}, 'medicaid-insurance-number': {'recall': 0.8899297423887588, 'accuracy': 0.915481832543444, 'f1_score': 0.8765859284890427, 'precision': 0.8636363636363636}, 'medicare-insurance-number': {'recall': 0.9224137931034483, 'accuracy': 0.9459888800635425, 'f1_score': 0.9042253521126761, 'precision': 0.8867403314917127}}, 'metadata': {'clearml_details': {'hf_model_id': '314e/dexit-aitest-vlm_qwen3_6-entity-integration', 'clearml_task_id': '137530ea3366487aa2d6d534f22d97e8', 'trained_entities': ['accession_number', 'collection_date', 'facility_address', 'facility_city', 'facility_name', 'facility_state', 'facility_zip', 'medicaid_insurance_number', 'medicare_insurance_number', 'other_insurance_name_1', 'other_insurance_name_2', 'other_insurance_number_1', 'other_insurance_number_2', 'patient_address', 'patient_city', 'patient_dob', 'patient_first_name', 'patient_last_name', 'patient_middle_name', 'patient_sex', 'patient_state', 'patient_zip', 'phone_number', 'provider_npi', 'sample_code', 'social_security_number'], 'new_modelversion_tag': 'v5.0.0', 'base_modelversion_tag': 'v4.0.0'}, 'hyperparameters': {'DATASET_PARAMS': {'DATASET__EVAL_S3_PATH': 's3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/eval.json', 'DATASET__TEST_S3_PATH': 's3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/test.json', 'DATASET__TRAIN_S3_PATH': 's3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/train.json', 'DATASET__MISCELLANEOUS_S3_PATH': 's3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/miscellaneous.json'}, 'MODEL_CONFIGURATION': {'SEED': 42, 'MODELNAME': 'vlm_qwen3_6', 'MODELTYPE': 'entity', 'ENVIRONMENT': 'integration', 'HF_MODEL_ID': '314e/dexit-aitest-vlm_qwen3_6-entity-integration', 'MODEL_FAMILY': 'qwen', 'ARTIFACTS_DIR': '/home/ubuntu/result', 'MODELVERSIONTAG': 'v4.0.0', 'HF_BASE_MODEL_ID': 'Qwen/Qwen3.6-27B', 'IMAGE_PATCH_SIZE': 16, 'THRESHOLD_OFFSET': 0.05, 'TRAIN_OOTB_MODEL': True, 'NEW_MODELVERSIONTAG': 'v5.0.0'}, 'MODEL_HYPERPARAMETERS': {'RANK': 128, 'ALPHA': 256, 'TRAIN_EPOCHS': 2, 'EVAL_BATCH_SIZE': 1, 'GRAD_ACCUM_STEPS': 4, 'TRAIN_BATCH_SIZE': 8, 'INFERENCE_BATCH_SIZE': 10}}, 'workflow_details': {'temporal_workflow_id': 'local-model-retraining-entity-eb64e687', 'skypilot_cluster_name': 'dexit-aitest-skypilot-entity-1787084040-969088', 'temporal_workflow_run_id': '01a01682-35fd-77f5-99b8-09424b615c58'}, 'modelconfig_snapshot': {'modelname': 'vlm_qwen3_6', 'modeltype': 'entity', 'deployment': {'auto_model_deployment_manual': False, 'auto_model_deployment_feedback': False}, 'feedback_config': {'bucket_low': 0.5, 'bucket_high': 0.7, 'auto_trigger_retraining': True, 'non_corrected_sample_ratio': 0.8, 'min_feedback_count_for_retraining': 1500, 'low_bucket_threshold_offset_multiplier': 0.5, 'auto_trigger_retraining_on_new_entities': False, 'high_bucket_threshold_offset_multiplier': 1.5, 'low_precision_bucket_corrected_sample_rate': 0.7, 'mid_precision_bucket_corrected_sample_rate': 0.7, 'high_precision_bucket_corrected_sample_rate': 0.8, 'low_precision_bucket_threshold_sample_count': 100}, 'training_config': {'seed': 42, 'dataset': {'eval_size': 0.1, 'test_size': 0.2, 'train_size': 0.7}, 'cloud_providers': [{'accelerators': ['A100-80GB:1', 'H100:1'], 'provider_name': 'runpod'}], 'hyperparameters': {'rank': 128, 'alpha': 256, 'train_epochs': 2, 'grad_accum_steps': 4, 'train_batch_size': 8}, 'hf_base_model_id': 'Qwen/Qwen3.6-27B', 'train_ootb_model': True, 'previous_datasets': {'eval': {'enabled': True, 'distribution': {'weights': {'n-1': 1.0}, 'strategy': 'equal', 'decay_factor': 0.8}, 'max_versions_to_include': 'all'}, 'test': {'enabled': True, 'distribution': {'weights': {'n-1': 1.0}, 'strategy': 'equal', 'decay_factor': 0.8}, 'max_versions_to_include': 'all'}, 'train': {'enabled': True, 'distribution': {'weights': {'n-1': 1.0}, 'strategy': 'equal', 'decay_factor': 0.8}, 'max_versions_to_include': 'all'}}}, 'threshold_offset': 0.05}}, 'deploymentstatus': None, 'deployeddate': None}
19
- 2026-08-18 20:30:24.114 | DEBUG | app.commonmlops:update_training_job_queue:940 - Update training job queue status code: 200, api_url: https://aitest.api.dexit.tech/api/v1/mlops/trainingjobqueue/06a665f8-ebd0-7481-8000-83304d751ace
20
- 2026-08-18 20:30:24.115 | SUCCESS | app.commonmlops:update_trained_entities:979 - Updated trained_entities on training job queue
21
- 2026-08-18 20:30:24.586 | INFO | app.tasks.entity.vlm.reports:upload_initial_artifacts:1703 - Uploaded intial artifacts to ClearML
22
- 2026-08-18 20:30:24.587 | INFO | __main__:_run_training_phase:635 - Moving into standard training loop
23
- 2026-08-18 20:30:24.588 | INFO | __main__:setup_directories:210 - Loading pre-split dataset statistics
24
- 2026-08-18 20:30:26.016 | INFO | __main__:load_presplit_statistics:187 - Found 4250 training samples
25
- 2026-08-18 20:30:26.419 | INFO | __main__:load_presplit_statistics:189 - Found 1245 test samples
26
- 2026-08-18 20:30:26.624 | INFO | __main__:load_presplit_statistics:191 - Found 563 validation samples
27
- 2026-08-18 20:30:26.625 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1044 -
28
  Dataset Split Statistics:
29
- 2026-08-18 20:30:26.625 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1045 - Total samples: 6058
30
- 2026-08-18 20:30:26.626 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1047 - Train set: 4250 (70.2%)
31
- 2026-08-18 20:30:26.626 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1048 - Test set: 1245 (20.6%)
32
- 2026-08-18 20:30:26.626 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1049 - Validation set: 563 (9.3%)
33
- 2026-08-18 20:30:26.628 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1058 - Split statistics saved to /home/ubuntu/result/split_statistics.json
34
- 2026-08-18 20:30:26.688 | INFO | __main__:standard_training_loop:332 - Initializing model
35
- 2026-08-18 20:30:26.688 | INFO | __main__:initialize_model:242 - Setting up model for retraining
36
- 2026-08-18 20:30:26.689 | INFO | __main__:initialize_model:245 - Training from base model: Qwen/Qwen3.6-27B
37
- 2026-08-18 20:30:26.689 | INFO | app.tasks.entity.vlm.model:__post_init__:299 - loading model Qwen/Qwen3.6-27B for training with unsloth
38
- 2026-08-18 20:32:44.596 | INFO | app.tasks.entity.vlm.model:calculate_max_steps:395 - num_samples: 4250, batch_size: 8, gradient_accumulation_steps: 4, epochs: 2
39
- 2026-08-18 20:32:44.597 | INFO | app.tasks.entity.vlm.model:calculate_max_steps:399 - calculating max steps
40
- 2026-08-18 20:32:44.643 | INFO | __main__:standard_training_loop:335 - Setting up trainer
41
- 2026-08-18 20:32:44.644 | INFO | __main__:create_trainer:284 - Datasets: 4250 train, 563 eval samples
42
- 2026-08-18 20:32:45.456 | INFO | __main__:standard_training_loop:338 - Starting training
43
- 2026-08-18 20:32:45.678 | INFO | app.tasks.entity.vlm.model:train_model:600 - everything is set up, launching training
44
- 2026-08-19 04:36:57.335 | INFO | app.tasks.entity.vlm.model:train_model:606 - training complete: TrainOutput(global_step=264, training_loss=0.23260276132460797, metrics={'train_runtime': 29020.6925, 'train_samples_per_second': 0.291, 'train_steps_per_second': 0.009, 'total_flos': 3.1571965250022564e+18, 'train_loss': 0.23260276132460797, 'epoch': 1.9849624060150375})
 
1
+ 2026-08-23 12:31:54.667 | INFO | app.tasks.entity.vlm.dataset:create:708 - Using pre-split dataset mode
2
+ 2026-08-23 12:31:54.688 | INFO | app.tasks.entity.vlm.dataset:_download_current_splits:1623 - Downloading train dataset from s3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/train.json
3
+ 2026-08-23 12:31:55.260 | INFO | app.tasks.entity.vlm.dataset:_download_split_documents:1480 - Processing %s %s/%s with %d documents
4
+ 2026-08-23 12:32:29.358 | INFO | app.tasks.entity.vlm.dataset:_download_current_splits:1623 - Downloading val dataset from s3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/eval.json
5
+ 2026-08-23 12:32:29.721 | INFO | app.tasks.entity.vlm.dataset:_download_split_documents:1480 - Processing %s %s/%s with %d documents
6
+ 2026-08-23 12:32:34.483 | INFO | app.tasks.entity.vlm.dataset:_download_current_splits:1623 - Downloading test dataset from s3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/test.json
7
+ 2026-08-23 12:32:34.827 | INFO | app.tasks.entity.vlm.dataset:_download_split_documents:1480 - Processing %s %s/%s with %d documents
8
+ 2026-08-23 12:32:47.975 | INFO | app.tasks.entity.vlm.dataset:_download_miscellaneous:1670 - Downloading miscellaneous data from s3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/miscellaneous.json
9
+ 2026-08-23 12:32:48.337 | INFO | app.tasks.entity.vlm.dataset:_download_previous_datasets:1579 - Found 0 previous dataset(s) to consolidate
10
+ 2026-08-23 12:32:48.338 | INFO | app.tasks.entity.vlm.dataset:download_presplit_datasets:1721 - Successfully downloaded all datasets to /home/ubuntu/dataset
11
+ 2026-08-23 12:33:11.558 | INFO | app.tasks.entity.vlm.dataset:remove_null_files:632 - Removed 2 files with null values
12
+ 2026-08-23 12:33:13.164 | INFO | app.tasks.entity.vlm.reports:get_distribution:1520 - Original number of ground truth files 6058
13
+ 2026-08-23 12:33:21.067 | INFO | app.tasks.entity.vlm.reports:get_distribution:1535 - After removing empty files 6058
14
+ 2026-08-23 12:33:21.324 | INFO | app.tasks.entity.vlm.reports:get_entity_distribution:1635 - Creating entity distribution for 6058 files
15
+ 2026-08-23 12:33:26.528 | DEBUG | app.commonmlops:get_oauth_token:834 - OAuth token request status: 200
16
+ 2026-08-23 12:33:26.808 | DEBUG | app.commonmlops:get_training_job_queue:891 - Get training job queue status code: 200, api_url: https://aitest.api.dexit.tech/api/v1/mlops/trainingjobqueue/06a665f8-ebd0-7481-8000-83304d751ace
17
+ 2026-08-23 12:33:27.032 | DEBUG | app.commonmlops:get_oauth_token:834 - OAuth token request status: 200
18
+ 2026-08-23 12:33:27.035 | DEBUG | app.commonmlops:update_training_job_queue:937 - Updating training job queue with merged payload: {'id': '06a665f8-ebd0-7481-8000-83304d751ace', 'modeltype': 'entity', 'modelname': 'vlm_qwen3_6', 'jobtype': 'feedback', 'basemodelversionid': '019f0d56-3844-7c0a-8fab-4aa6a6d1f12e', 'datasetid': '06a665f5-06bf-7f15-8000-acbbe709e051', 'status': 'triggered', 'createddate': '2026-07-26T19:27:10.738422Z', 'createdby': '00000000-0000-0000-0000-000000000000', 'startdate': '2026-08-23T12:31:23.779701Z', 'enddate': None, 'accuracyreport': {'overall': {'recall': 0.9489486112672889, 'f1_score': 0.9511143670226255, 'precision': 0.953290031064671}, 'patient': {'recall': 0.9570468593654881, 'accuracy': 0.965850180851059, 'f1_score': 0.9578867442511313, 'precision': 0.9588553005522813}, 'facility': {'recall': 0.9681215596546779, 'accuracy': 0.9468811825459762, 'f1_score': 0.9706030656585861, 'precision': 0.9731011770220469}, 'patient-dob': {'recall': 0.9812606473594548, 'accuracy': 0.9683042789223455, 'f1_score': 0.9829351535836178, 'precision': 0.9846153846153847}, 'patient-sex': {'recall': 0.9982847341337907, 'accuracy': 0.9967871485943776, 'f1_score': 0.9982847341337907, 'precision': 0.9982847341337907}, 'patient-zip': {'recall': 0.9623762376237623, 'accuracy': 0.9839486356340289, 'f1_score': 0.9798387096774194, 'precision': 0.997946611909651}, 'patient.dob': {'recall': 0.9821124361158433, 'accuracy': 0.9706582077716098, 'f1_score': 0.9842082799829279, 'precision': 0.9863130881094953}, 'patient.sex': {'recall': 0.9991423670668954, 'accuracy': 0.9983935742971888, 'f1_score': 0.9991423670668954, 'precision': 0.9991423670668954}, 'patient.zip': {'recall': 0.9663366336633663, 'accuracy': 0.9847512038523274, 'f1_score': 0.9809045226130653, 'precision': 0.9959183673469387}, 'sample-code': {'recall': 0.9737609329446064, 'accuracy': 0.9545454545454546, 'f1_score': 0.9732880038853812, 'precision': 0.9728155339805825}, 'facility-zip': {'recall': 0.9796782387806944, 'accuracy': 0.9728434504792333, 'f1_score': 0.985519591141397, 'precision': 0.9914310197086547}, 'facility.zip': {'recall': 0.9847586790855207, 'accuracy': 0.9808, 'f1_score': 0.9897872340425531, 'precision': 0.9948674080410608}, 'patient-city': {'recall': 0.9756592292089249, 'accuracy': 0.9864, 'f1_score': 0.982635342185904, 'precision': 0.9897119341563786}, 'patient.city': {'recall': 0.9817444219066938, 'accuracy': 0.988, 'f1_score': 0.9847405900305187, 'precision': 0.9877551020408163}, 'phone-number': {'recall': 0.9784172661870504, 'accuracy': 0.9847756410256411, 'f1_score': 0.9662522202486679, 'precision': 0.9543859649122807}, 'provider-npi': {'recall': 1.0, 'accuracy': 0.9991546914623838, 'f1_score': 0.9995147986414362, 'precision': 0.9990300678952473}, 'facility-city': {'recall': 0.9931682322801024, 'accuracy': 0.9903691813804173, 'f1_score': 0.9948674080410608, 'precision': 0.9965724078834619}, 'facility-name': {'recall': 0.9692570452604612, 'accuracy': 0.9460093896713615, 'f1_score': 0.9705002137665669, 'precision': 0.9717465753424658}, 'facility.city': {'recall': 0.9965841161400513, 'accuracy': 0.9967871485943776, 'f1_score': 0.998289136013687, 'precision': 1.0}, 'facility.name': {'recall': 0.9624252775405636, 'accuracy': 0.9339035769828927, 'f1_score': 0.9636596836254809, 'precision': 0.9648972602739726}, 'patient-state': {'recall': 0.7729007633587787, 'accuracy': 0.8273062730627306, 'f1_score': 0.7758620689655172, 'precision': 0.7788461538461539}, 'patient.state': {'recall': 0.8530534351145038, 'accuracy': 0.8843226788432268, 'f1_score': 0.8546845124282982, 'precision': 0.8563218390804598}, 'facility-state': {'recall': 0.8662691652470187, 'accuracy': 0.7794540229885057, 'f1_score': 0.8688594617684751, 'precision': 0.87146529562982}, 'facility.state': {'recall': 0.9335604770017035, 'accuracy': 0.8867781155015197, 'f1_score': 0.936351986330628, 'precision': 0.9391602399314481}, 'collection-date': {'recall': 0.8938053097345132, 'accuracy': 0.9175597691673537, 'f1_score': 0.9099099099099099, 'precision': 0.926605504587156}, 'patient-address': {'recall': 0.8693877551020408, 'accuracy': 0.9042145593869731, 'f1_score': 0.872057318321392, 'precision': 0.8747433264887063}, 'patient.address': {'recall': 0.9, 'accuracy': 0.9255813953488372, 'f1_score': 0.901840490797546, 'precision': 0.9036885245901639}, 'accession-number': {'recall': 0.9967948717948718, 'accuracy': 0.9991546914623838, 'f1_score': 0.9983948635634029, 'precision': 1.0}, 'facility-address': {'recall': 0.9513236549957301, 'accuracy': 0.9173106646058733, 'f1_score': 0.9541755888650965, 'precision': 0.9570446735395189}, 'facility.address': {'recall': 0.9632792485055508, 'accuracy': 0.9361370716510904, 'f1_score': 0.9649272882805817, 'precision': 0.9665809768637532}, 'patient-last-name': {'recall': 0.9879518072289156, 'accuracy': 0.9785202863961814, 'f1_score': 0.9891435464414957, 'precision': 0.9903381642512077}, 'patient.last-name': {'recall': 0.9879518072289156, 'accuracy': 0.9785202863961814, 'f1_score': 0.9891435464414957, 'precision': 0.9903381642512077}, 'patient-first-name': {'recall': 0.9927652733118971, 'accuracy': 0.9872102318145484, 'f1_score': 0.9935639581657282, 'precision': 0.9943639291465378}, 'patient.first-name': {'recall': 0.9911575562700965, 'accuracy': 0.9848242811501597, 'f1_score': 0.9923541247484909, 'precision': 0.9935535858178888}, 'patient-middle-name': {'recall': 0.9182692307692307, 'accuracy': 0.9641434262948207, 'f1_score': 0.8946135831381734, 'precision': 0.8721461187214612}, 'patient.middle-name': {'recall': 0.9519230769230769, 'accuracy': 0.9776, 'f1_score': 0.9339622641509433, 'precision': 0.9166666666666666}, 'other-insurance-name-1': {'recall': 0.7476190476190476, 'accuracy': 0.8555900621118012, 'f1_score': 0.7714987714987716, 'precision': 0.7969543147208121}, 'other-insurance-name-2': {'recall': 1.0, 'accuracy': 0.9975903614457832, 'f1_score': 0.4, 'precision': 0.25}, 'social-security-number': {'recall': 0.9210526315789473, 'accuracy': 0.9847634322373697, 'f1_score': 0.7865168539325842, 'precision': 0.6862745098039216}, 'other-insurance-number-1': {'recall': 0.8333333333333334, 'accuracy': 0.9074221867517956, 'f1_score': 0.857843137254902, 'precision': 0.8838383838383839}, 'other-insurance-number-2': {'recall': 1.0, 'accuracy': 0.9975903614457832, 'f1_score': 0.4, 'precision': 0.25}, 'medicaid-insurance-number': {'recall': 0.8899297423887588, 'accuracy': 0.915481832543444, 'f1_score': 0.8765859284890427, 'precision': 0.8636363636363636}, 'medicare-insurance-number': {'recall': 0.9224137931034483, 'accuracy': 0.9459888800635425, 'f1_score': 0.9042253521126761, 'precision': 0.8867403314917127}}, 'metadata': {'clearml_details': {'hf_model_id': '314e/dexit-aitest-vlm_qwen3_6-entity-integration', 'clearml_task_id': 'aca31aa9c3624759b08e0c983307a86e', 'trained_entities': ['accession_number', 'collection_date', 'facility_address', 'facility_city', 'facility_name', 'facility_state', 'facility_zip', 'medicaid_insurance_number', 'medicare_insurance_number', 'other_insurance_name_1', 'other_insurance_name_2', 'other_insurance_number_1', 'other_insurance_number_2', 'patient_address', 'patient_city', 'patient_dob', 'patient_first_name', 'patient_last_name', 'patient_middle_name', 'patient_sex', 'patient_state', 'patient_zip', 'phone_number', 'provider_npi', 'sample_code', 'social_security_number'], 'new_modelversion_tag': 'v6.0.0', 'base_modelversion_tag': 'v4.0.0'}, 'hyperparameters': {'DATASET_PARAMS': {'DATASET__EVAL_S3_PATH': 's3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/eval.json', 'DATASET__TEST_S3_PATH': 's3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/test.json', 'DATASET__TRAIN_S3_PATH': 's3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/train.json', 'DATASET__MISCELLANEOUS_S3_PATH': 's3://aitest-dexit-tech/data/__mlops__/dataset/entity/06a665f5-06bf-7f15-8000-acbbe709e051/miscellaneous.json'}, 'MODEL_CONFIGURATION': {'SEED': 42, 'MODELNAME': 'vlm_qwen3_6', 'MODELTYPE': 'entity', 'ENVIRONMENT': 'integration', 'HF_MODEL_ID': '314e/dexit-aitest-vlm_qwen3_6-entity-integration', 'MODEL_FAMILY': 'qwen', 'ARTIFACTS_DIR': '/home/ubuntu/result', 'MODELVERSIONTAG': 'v4.0.0', 'HF_BASE_MODEL_ID': 'Qwen/Qwen3.6-27B', 'IMAGE_PATCH_SIZE': 16, 'THRESHOLD_OFFSET': 0.05, 'TRAIN_OOTB_MODEL': True, 'NEW_MODELVERSIONTAG': 'v6.0.0'}, 'MODEL_HYPERPARAMETERS': {'RANK': 256, 'ALPHA': 512, 'TRAIN_EPOCHS': 4, 'EVAL_BATCH_SIZE': 1, 'GRAD_ACCUM_STEPS': 4, 'TRAIN_BATCH_SIZE': 8, 'INFERENCE_BATCH_SIZE': 10}}, 'workflow_details': {'temporal_workflow_id': 'local-model-retraining-entity-fde59196', 'skypilot_cluster_name': 'dexit-aitest-skypilot-entity-1787487611-069422', 'temporal_workflow_run_id': '01a02e90-340f-7c62-a77a-e4d517ae68fe'}, 'modelconfig_snapshot': {'modelname': 'vlm_qwen3_6', 'modeltype': 'entity', 'deployment': {'auto_model_deployment_manual': False, 'auto_model_deployment_feedback': False}, 'feedback_config': {'bucket_low': 0.5, 'bucket_high': 0.7, 'auto_trigger_retraining': True, 'non_corrected_sample_ratio': 0.8, 'min_feedback_count_for_retraining': 1500, 'low_bucket_threshold_offset_multiplier': 0.5, 'auto_trigger_retraining_on_new_entities': False, 'high_bucket_threshold_offset_multiplier': 1.5, 'low_precision_bucket_corrected_sample_rate': 0.7, 'mid_precision_bucket_corrected_sample_rate': 0.7, 'high_precision_bucket_corrected_sample_rate': 0.8, 'low_precision_bucket_threshold_sample_count': 100}, 'training_config': {'seed': 42, 'dataset': {'eval_size': 0.1, 'test_size': 0.2, 'train_size': 0.7}, 'cloud_providers': [{'accelerators': ['A100-80GB:1', 'H100:1'], 'provider_name': 'runpod'}], 'hyperparameters': {'rank': 256, 'alpha': 512, 'train_epochs': 4, 'grad_accum_steps': 4, 'train_batch_size': 8}, 'hf_base_model_id': 'Qwen/Qwen3.6-27B', 'train_ootb_model': True, 'previous_datasets': {'eval': {'enabled': True, 'distribution': {'weights': {'n-1': 1.0}, 'strategy': 'equal', 'decay_factor': 0.8}, 'max_versions_to_include': 'all'}, 'test': {'enabled': True, 'distribution': {'weights': {'n-1': 1.0}, 'strategy': 'equal', 'decay_factor': 0.8}, 'max_versions_to_include': 'all'}, 'train': {'enabled': True, 'distribution': {'weights': {'n-1': 1.0}, 'strategy': 'equal', 'decay_factor': 0.8}, 'max_versions_to_include': 'all'}}}, 'threshold_offset': 0.05}}, 'deploymentstatus': None, 'deployeddate': None}
19
+ 2026-08-23 12:33:27.338 | DEBUG | app.commonmlops:update_training_job_queue:940 - Update training job queue status code: 200, api_url: https://aitest.api.dexit.tech/api/v1/mlops/trainingjobqueue/06a665f8-ebd0-7481-8000-83304d751ace
20
+ 2026-08-23 12:33:27.340 | SUCCESS | app.commonmlops:update_trained_entities:979 - Updated trained_entities on training job queue
21
+ 2026-08-23 12:33:27.803 | INFO | app.tasks.entity.vlm.reports:upload_initial_artifacts:1703 - Uploaded intial artifacts to ClearML
22
+ 2026-08-23 12:33:27.804 | INFO | __main__:_run_training_phase:635 - Moving into standard training loop
23
+ 2026-08-23 12:33:27.804 | INFO | __main__:setup_directories:210 - Loading pre-split dataset statistics
24
+ 2026-08-23 12:33:29.238 | INFO | __main__:load_presplit_statistics:187 - Found 4250 training samples
25
+ 2026-08-23 12:33:29.694 | INFO | __main__:load_presplit_statistics:189 - Found 1245 test samples
26
+ 2026-08-23 12:33:29.941 | INFO | __main__:load_presplit_statistics:191 - Found 563 validation samples
27
+ 2026-08-23 12:33:29.942 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1044 -
28
  Dataset Split Statistics:
29
+ 2026-08-23 12:33:29.943 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1045 - Total samples: 6058
30
+ 2026-08-23 12:33:29.943 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1047 - Train set: 4250 (70.2%)
31
+ 2026-08-23 12:33:29.943 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1048 - Test set: 1245 (20.6%)
32
+ 2026-08-23 12:33:29.944 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1049 - Validation set: 563 (9.3%)
33
+ 2026-08-23 12:33:29.946 | INFO | app.tasks.entity.vlm.dataset:print_split_statistics:1058 - Split statistics saved to /home/ubuntu/result/split_statistics.json
34
+ 2026-08-23 12:33:30.006 | INFO | __main__:standard_training_loop:332 - Initializing model
35
+ 2026-08-23 12:33:30.007 | INFO | __main__:initialize_model:242 - Setting up model for retraining
36
+ 2026-08-23 12:33:30.007 | INFO | __main__:initialize_model:245 - Training from base model: Qwen/Qwen3.6-27B
37
+ 2026-08-23 12:33:30.008 | INFO | app.tasks.entity.vlm.model:__post_init__:321 - loading model Qwen/Qwen3.6-27B for training with unsloth
38
+ 2026-08-23 12:41:37.011 | INFO | app.tasks.entity.vlm.model:calculate_max_steps:417 - num_samples: 4250, batch_size: 8, gradient_accumulation_steps: 4, epochs: 4
39
+ 2026-08-23 12:41:37.012 | INFO | app.tasks.entity.vlm.model:calculate_max_steps:421 - calculating max steps
40
+ 2026-08-23 12:41:37.070 | INFO | __main__:standard_training_loop:335 - Setting up trainer
41
+ 2026-08-23 12:41:37.071 | INFO | __main__:create_trainer:284 - Datasets: 4250 train, 563 eval samples
42
+ 2026-08-23 12:41:37.969 | INFO | __main__:standard_training_loop:338 - Starting training
43
+ 2026-08-23 12:41:38.170 | INFO | app.tasks.entity.vlm.model:train_model:624 - everything is set up, launching training
 
ground_truth_data.json CHANGED
The diff for this file is too large to render. See raw diff
 
gt_json.json CHANGED
The diff for this file is too large to render. See raw diff
 
runs/Aug23_12-41-40_3c59ea7c2175/events.out.tfevents.1787488900.3c59ea7c2175.10055.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3f589e175ac602388bad5f906ffebbfb672b47c028e34471016974b0f482bf8
3
+ size 15415
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d2d221a16338752d2ce8afdb3b80ffc934a64b833121319092c06e00b7cae12a
3
  size 5905
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e8f59062181d3f8f0b4de9b3e54dd0a9c308d856aaca622e8009e97da3e6d1f
3
  size 5905