resume-parser / evaluation_summary.json
md-zain's picture
Simplify chart to quantized pre-fine-tuning and post-fine-tuning
413705c verified
Raw History Blame Contribute Delete
3.71 kB
{
"model_repo": "md-zain/resume-parser",
"evaluated_model_commit": "86649ee4bc8b06987bbd98462b009892183b2e2a",
"source_checkpoint_commit": "4128d0e4627ee5d71f88c4e9f14efc4f147f64c2",
"evaluation_status": "completed_400_resume_sample",
"dataset_repo": "md-zain/resume-parser-dataset",
"split": "test",
"source_split_sha256": "f01b3831d19f8528683c629521c7d618a439aedc050accaf98028d6bd2471f9a",
"sampling": "uniform random sample without replacement; seed 42; original split row order",
"generation": {
"max_new_tokens": 2048,
"context_length": 32768,
"do_sample": false,
"load_in_4bit": true,
"compute_dtype": "float16",
"prompt_style": "compact",
"stop": "EOS or first complete JSON checked every 16 tokens"
},
"prompt_sha256": "ca308193030079391c13bfc8772649004c4bf999d0861f6ab47af004264423c6",
"schema_sha256": "45bd363dd4c97a256a48163084e26b2c5f389c377ff08751a89f35226f5cbb61",
"counts": {
"items_evaluated": 400,
"valid_targets": 400,
"json_valid_count": 357,
"schema_valid_count": 356,
"scored_count": 356,
"source_rows": 1463,
"expected_rows": 400,
"completed_rows": 400,
"sample_seed": 42,
"retryable_error_count": 0,
"output_limit_count": 43
},
"scores": {
"mean_programmatic_score": 0.908859,
"mean_semantic_score": 0.806636,
"mean_overall_score": 0.857748,
"mean_identity_score": 0.978901,
"overall_score_including_invalid_predictions": 0.763395,
"json_valid_rate": 0.8925,
"schema_valid_rate": 0.89
},
"status_counts": {
"SCORED": 356,
"INVALID_JSON": 43,
"INVALID_PREDICTION_SCHEMA": 1
},
"scoring": "Step 04 Evaluator: deterministic field comparisons plus TypeSafe AI semantic scoring",
"score_interpretation": {
"mean_overall_score": "Mean among the 356 fully scored resumes.",
"overall_score_including_invalid_predictions": "Mean over all 400 valid targets, with failed outputs scored as zero."
},
"limitations": [
"This is a 400-resume sample of 1463 reserved test resumes, not full-split or universal accuracy.",
"All 43 invalid JSON outputs hit the declared 2048-token generation ceiling.",
"Historical pre-evaluation runs each used 5 resumes with different samples/prompts/settings; comparison is descriptive only.",
"The Quantized label follows the historical filename; exact precision is not embedded in that report."
],
"historical_comparison": {
"runs": [
{
"name": "Pre-fine-tuning (Quantized)",
"evaluated": 5,
"scored": 3,
"json_valid_count": 4,
"schema_valid_count": 3,
"overall": 0.719074,
"programmatic": 0.774263,
"semantic": 0.663885,
"identity": 1.0,
"overall_all": 0.431444,
"json_valid": 0.8,
"schema_valid": 0.6
},
{
"name": "Post-fine-tuning",
"evaluated": 400,
"scored": 356,
"json_valid_count": 357,
"schema_valid_count": 356,
"overall": 0.857748,
"programmatic": 0.908859,
"semantic": 0.806636,
"identity": 0.978901,
"overall_all": 0.763395,
"json_valid": 0.8925,
"schema_valid": 0.89
}
],
"difference_vs_quantized_report_percentage_points": {
"overall_all": 33.195100000000004,
"overall": 13.867399999999996,
"programmatic": 13.459599999999995,
"semantic": 14.275100000000007,
"identity": -2.109899999999998,
"json_valid": 9.249999999999991,
"schema_valid": 29.000000000000004
},
"matched_evaluation_conditions": false
},
"wandb_run": "https://wandb.ai/zainmemon010-personal/resume-parser/runs/do783w9g"
}