{ "model_repo": "md-zain/resume-parser", "evaluated_model_commit": "86649ee4bc8b06987bbd98462b009892183b2e2a", "source_checkpoint_commit": "4128d0e4627ee5d71f88c4e9f14efc4f147f64c2", "evaluation_status": "completed_400_resume_sample", "dataset_repo": "md-zain/resume-parser-dataset", "split": "test", "source_split_sha256": "f01b3831d19f8528683c629521c7d618a439aedc050accaf98028d6bd2471f9a", "sampling": "uniform random sample without replacement; seed 42; original split row order", "generation": { "max_new_tokens": 2048, "context_length": 32768, "do_sample": false, "load_in_4bit": true, "compute_dtype": "float16", "prompt_style": "compact", "stop": "EOS or first complete JSON checked every 16 tokens" }, "prompt_sha256": "ca308193030079391c13bfc8772649004c4bf999d0861f6ab47af004264423c6", "schema_sha256": "45bd363dd4c97a256a48163084e26b2c5f389c377ff08751a89f35226f5cbb61", "counts": { "items_evaluated": 400, "valid_targets": 400, "json_valid_count": 357, "schema_valid_count": 356, "scored_count": 356, "source_rows": 1463, "expected_rows": 400, "completed_rows": 400, "sample_seed": 42, "retryable_error_count": 0, "output_limit_count": 43 }, "scores": { "mean_programmatic_score": 0.908859, "mean_semantic_score": 0.806636, "mean_overall_score": 0.857748, "mean_identity_score": 0.978901, "overall_score_including_invalid_predictions": 0.763395, "json_valid_rate": 0.8925, "schema_valid_rate": 0.89 }, "status_counts": { "SCORED": 356, "INVALID_JSON": 43, "INVALID_PREDICTION_SCHEMA": 1 }, "scoring": "Step 04 Evaluator: deterministic field comparisons plus TypeSafe AI semantic scoring", "score_interpretation": { "mean_overall_score": "Mean among the 356 fully scored resumes.", "overall_score_including_invalid_predictions": "Mean over all 400 valid targets, with failed outputs scored as zero." }, "limitations": [ "This is a 400-resume sample of 1463 reserved test resumes, not full-split or universal accuracy.", "All 43 invalid JSON outputs hit the declared 2048-token generation ceiling.", "Historical pre-evaluation runs each used 5 resumes with different samples/prompts/settings; comparison is descriptive only.", "The Quantized label follows the historical filename; exact precision is not embedded in that report." ], "historical_comparison": { "runs": [ { "name": "Pre-fine-tuning (Quantized)", "evaluated": 5, "scored": 3, "json_valid_count": 4, "schema_valid_count": 3, "overall": 0.719074, "programmatic": 0.774263, "semantic": 0.663885, "identity": 1.0, "overall_all": 0.431444, "json_valid": 0.8, "schema_valid": 0.6 }, { "name": "Post-fine-tuning", "evaluated": 400, "scored": 356, "json_valid_count": 357, "schema_valid_count": 356, "overall": 0.857748, "programmatic": 0.908859, "semantic": 0.806636, "identity": 0.978901, "overall_all": 0.763395, "json_valid": 0.8925, "schema_valid": 0.89 } ], "difference_vs_quantized_report_percentage_points": { "overall_all": 33.195100000000004, "overall": 13.867399999999996, "programmatic": 13.459599999999995, "semantic": 14.275100000000007, "identity": -2.109899999999998, "json_valid": 9.249999999999991, "schema_valid": 29.000000000000004 }, "matched_evaluation_conditions": false }, "wandb_run": "https://wandb.ai/zainmemon010-personal/resume-parser/runs/do783w9g" }