| """Preprocess materialize stage — counts (raw flat_skills), inline experience_band, clean signals, text.""" |
|
|
| from __future__ import annotations |
|
|
| import copy |
| from pathlib import Path |
|
|
| import orjson |
|
|
| from preprocess import DATASET_AS_OF, load_candidates, materialize |
|
|
|
|
| def _record() -> dict: |
| """A populated candidate (keeps every nested column struct-typed).""" |
| return { |
| "candidate_id": "CAND_0000001", |
| "profile": {"years_of_experience": 7.0, "current_title": "AI Engineer", |
| "current_company": "Acme", "current_industry": "Tech"}, |
| "career_history": [ |
| {"title": "AI Engineer", "company": "Acme", "industry": "Tech", "company_size": "51-200", |
| "duration_months": 40, "is_current": True, "start_date": "2021-01-01", |
| "end_date": None, "description": "ranking systems"} |
| ], |
| "education": [{"institution": "IIT", "degree": "BTech", "field_of_study": "CS", |
| "start_year": 2014, "end_year": 2018, "tier": "tier_1"}], |
| "skills": [ |
| {"name": "Python", "proficiency": "expert", "endorsements": 9, "duration_months": 60}, |
| {"name": "SQL", "proficiency": "beginner", "endorsements": 1, "duration_months": 12}, |
| ], |
| "certifications": [{"name": "AWS SAA", "issuer": "AWS", "year": 2022}], |
| "redrob_signals": { |
| "github_activity_score": 70, "offer_acceptance_rate": -1, "notice_period_days": 15, |
| "expected_salary_range_inr_lpa": {"min": 20, "max": 40}, |
| "skill_assessment_scores": {"Python": 88.0}, |
| }, |
| } |
|
|
|
|
| def _materialize(tmp_path: Path, records: list[dict]): |
| path = tmp_path / "c.jsonl" |
| path.write_bytes(b"\n".join(orjson.dumps(r) for r in records)) |
| return materialize(load_candidates(path), DATASET_AS_OF) |
|
|
|
|
| def test_experience_band_is_computed_inline(tmp_path: Path) -> None: |
| bands = [] |
| for yoe in (2.0, 5.0, 8.0, 12.0, 20.0): |
| record = _record() |
| record["profile"]["years_of_experience"] = yoe |
| bands.append(_materialize(tmp_path, [record]).row(0, named=True)["experience_band"]) |
| assert bands == ["junior", "mid", "senior", "lead", "principal"] |
|
|
|
|
| def test_counts_use_raw_flat_skills(tmp_path: Path) -> None: |
| row = _materialize(tmp_path, [_record()]).row(0, named=True) |
| assert row["num_skills"] == 2 |
| assert row["num_advanced_skills"] == 1 and row["num_expert_skills"] == 1 |
| assert row["num_roles"] == 1 and row["num_companies"] == 1 |
| assert row["num_assessments_taken"] == 1 and row["max_skill_assessment"] == 88.0 |
|
|
|
|
| def test_sentinel_signals_are_cleaned(tmp_path: Path) -> None: |
| row = _materialize(tmp_path, [_record()]).row(0, named=True) |
| assert row["has_github"] is True and row["github_score_clean"] == 70.0 |
| assert row["has_offer_history"] is False and row["offer_acceptance_rate_clean"] is None |
| assert row["salary_mid"] == 30.0 and row["is_available_soon"] is True |
|
|
|
|
| def test_embedded_text_is_built(tmp_path: Path) -> None: |
| row = _materialize(tmp_path, [_record()]).row(0, named=True) |
| assert row["text"] == row["summary_text"] |
| assert "AI Engineer" in row["text"] and "senior-band" in row["text"] |
| assert len(row["embed_source_hash"]) == 16 |
|
|
|
|
| def test_materialize_never_drops(tmp_path: Path) -> None: |
| second = copy.deepcopy(_record()) |
| second["candidate_id"] = "CAND_0000002" |
| assert _materialize(tmp_path, [_record(), second]).height == 2 |
|
|
