Spaces:
Sleeping
Sleeping
| # ============================================================================= | |
| # engine/extractor.py — Structured Field Extraction from Resume Text | |
| # ============================================================================= | |
| # PURPOSE: | |
| # Takes raw resume text (output of parser.py) and extracts structured fields. | |
| # This is the most important file in the engine — everything downstream | |
| # (scoring, explainability) depends on what this file extracts. | |
| # | |
| # WHAT IT EXTRACTS: | |
| # - Candidate name, email, phone | |
| # - Technical skills (matched against taxonomy) | |
| # - Soft skills | |
| # - Work experience (years, roles, companies, tenure, stability) | |
| # - Education (degree, field, institution) | |
| # - Projects, certifications, hobbies | |
| # - Recency flag (how recently key skills were used) | |
| # | |
| # MAIN FUNCTION (only one you call from outside): | |
| # build_resume_record(file_name, raw_text) → dict | |
| # This calls all other functions and returns one complete DB-ready dict. | |
| # | |
| # USAGE: | |
| # from engine.extractor import build_resume_record | |
| # record = build_resume_record("resume_1.pdf", raw_text) | |
| # ============================================================================= | |
| import os | |
| import re | |
| import sys | |
| import json | |
| import time | |
| from datetime import datetime | |
| from dateutil import parser as dateparser | |
| from dateutil.relativedelta import relativedelta | |
| # Add project root to path | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| import config | |
| # ML skill model — trained on job postings dataset | |
| # Falls back silently if model hasn't been trained yet | |
| try: | |
| from train_skill_model import predict_skills_for_text as _ml_predict | |
| _ML_MODEL_OK = True | |
| except ImportError: | |
| _ML_MODEL_OK = False | |
| def _ml_predict(*args, **kwargs): return [] | |
| # spaCy — load once at module level so it isn't reloaded for every resume | |
| try: | |
| import spacy | |
| _NLP = spacy.load("en_core_web_sm") | |
| _SPACY_OK = True | |
| except (ImportError, OSError): | |
| _NLP = None | |
| _SPACY_OK = False | |
| print("[Extractor] WARNING: spaCy model not found. " | |
| "Run: python -m spacy download en_core_web_sm") | |
| # ============================================================================= | |
| # ESCO + JobBERT-v2 SKILL INDEX | |
| # ============================================================================= | |
| # Built by: python build_esco_index.py | |
| # Three artefacts in models/: | |
| # esco_embeddings.npy — (N, dim) L2-normalised float32 | |
| # esco_skills.json — {labels, altLabels, skillType, model, …} | |
| # esco_index_report.json | |
| # | |
| # If any artefact is missing, the extractor silently falls back to the legacy | |
| # taxonomy match + TF-IDF model — so the pipeline never crashes mid-deployment. | |
| _ESCO_OK = False | |
| _ESCO_MODEL = None | |
| _ESCO_EMBEDS = None # numpy (N, dim) | |
| _ESCO_META = None # parsed esco_skills.json | |
| _ESCO_SOFT_IDX = set() # int indices of transversal/soft ESCO skills | |
| # Optional binary "is this label soft?" classifier head trained on top of | |
| # JobBERT-v2. Loaded lazily; if absent, runtime falls back to rule-based | |
| # routing (denylist + ESCO transversal lookup). | |
| _SOFTNESS_CLF = None | |
| _SOFTNESS_THRESHOLD = 0.7 # conservative — only fire on confident soft hits | |
| # LLM-classified overrides for every skill seen in resume_data.csv. Produced | |
| # by llm_classify_skills.py. Authoritative when present — overrides denylist | |
| # and ESCO transversal lookup. Maps lowercase label → {"category", "...}. | |
| _SKILL_OVERRIDES: dict[str, dict] = {} | |
| def _load_esco_index() -> None: | |
| """Best-effort load of the ESCO embedding cache. Idempotent.""" | |
| global _ESCO_OK, _ESCO_MODEL, _ESCO_EMBEDS, _ESCO_META, _ESCO_SOFT_IDX | |
| import numpy as _np | |
| _model_dir = os.path.join(config.BASE_DIR, "models") | |
| _embed_path = os.path.join(_model_dir, "esco_embeddings.npy") | |
| _labels_path = os.path.join(_model_dir, "esco_skills.json") | |
| if not (os.path.exists(_embed_path) and os.path.exists(_labels_path)): | |
| return | |
| try: | |
| from sentence_transformers import SentenceTransformer | |
| _ESCO_EMBEDS = _np.load(_embed_path) | |
| with open(_labels_path, encoding="utf-8") as f: | |
| _ESCO_META = json.load(f) | |
| model_name = _ESCO_META.get("model", "TechWolf/JobBERT-v2") | |
| _ESCO_MODEL = SentenceTransformer(model_name) | |
| _ESCO_OK = True | |
| print(f"[Extractor] ESCO index loaded — " | |
| f"{_ESCO_EMBEDS.shape[0]:,} skills via {model_name}") | |
| # Map transversal-skill conceptUris to indices into _ESCO_EMBEDS so we | |
| # can route soft hits at query time without rebuilding the index. | |
| _transversal_path = os.path.join(config.BASE_DIR, "data", | |
| "esco_transversal_uris.txt") | |
| if os.path.exists(_transversal_path): | |
| soft_uris = set() | |
| with open(_transversal_path, encoding="utf-8") as f: | |
| for line in f: | |
| line = line.strip() | |
| if not line or line.startswith("#"): | |
| continue | |
| soft_uris.add(line.split("\t", 1)[0]) | |
| uris = _ESCO_META.get("conceptUri", []) | |
| _ESCO_SOFT_IDX = {i for i, u in enumerate(uris) if u in soft_uris} | |
| print(f"[Extractor] Transversal index loaded — " | |
| f"{len(_ESCO_SOFT_IDX):,} soft-skill URIs") | |
| except Exception as e: | |
| print(f"[Extractor] ESCO index load failed: {type(e).__name__}: {e}") | |
| _ESCO_OK = False | |
| _load_esco_index() | |
| def _load_softness_classifier() -> None: | |
| """Lazy-load the optional JobBERT softness classifier. Idempotent.""" | |
| global _SOFTNESS_CLF | |
| if _SOFTNESS_CLF is not None: | |
| return | |
| import pickle | |
| _path = os.path.join(config.BASE_DIR, "models", "softness_classifier.pkl") | |
| if not os.path.exists(_path): | |
| return | |
| try: | |
| with open(_path, "rb") as f: | |
| _SOFTNESS_CLF = pickle.load(f) | |
| m = _SOFTNESS_CLF.get("metrics", {}) | |
| print(f"[Extractor] Softness classifier loaded — " | |
| f"F1(soft)={m.get('f1_soft', 0):.3f}, " | |
| f"threshold={_SOFTNESS_THRESHOLD}") | |
| except Exception as e: | |
| print(f"[Extractor] Softness classifier load failed: " | |
| f"{type(e).__name__}: {e}") | |
| _load_softness_classifier() | |
| def _load_skill_overrides() -> None: | |
| """ | |
| Load data/skill_softness_overrides.csv if present. This is the LLM- | |
| classified routing table for every skill seen in resume_data.csv — | |
| authoritative when present, overrides denylist + ESCO transversal lookup. | |
| """ | |
| import csv as _csv | |
| global _SKILL_OVERRIDES | |
| path = os.path.join(config.BASE_DIR, "data", "skill_softness_overrides.csv") | |
| if not os.path.exists(path): | |
| return | |
| try: | |
| with open(path, encoding="utf-8") as f: | |
| for row in _csv.DictReader(f): | |
| lbl = (row.get("label") or "").strip() | |
| cat = (row.get("category") or "").strip().lower() | |
| if lbl and cat in {"soft", "hard", "language", "other"}: | |
| _SKILL_OVERRIDES[lbl.lower()] = { | |
| "category": cat, | |
| "confidence": (row.get("confidence") or "").strip().lower(), | |
| "human_reviewed": (row.get("human_reviewed") or "").strip(), | |
| } | |
| from collections import Counter as _Counter | |
| cats = _Counter(v["category"] for v in _SKILL_OVERRIDES.values()) | |
| print(f"[Extractor] Skill overrides loaded — " | |
| f"{len(_SKILL_OVERRIDES):,} labels ({dict(cats)})") | |
| except Exception as e: | |
| print(f"[Extractor] Skill overrides load failed: " | |
| f"{type(e).__name__}: {e}") | |
| _load_skill_overrides() | |
| def _classify_softness(labels: list[str]) -> dict[str, float]: | |
| """ | |
| Return {label: P(soft)} for each input label. Uses JobBERT-v2 encoder | |
| that's already loaded for ESCO. If the classifier isn't available, | |
| returns an empty dict (caller falls back to rule-based routing). | |
| """ | |
| if _SOFTNESS_CLF is None or _ESCO_MODEL is None or not labels: | |
| return {} | |
| import numpy as _np | |
| enc = _ESCO_MODEL.encode( | |
| labels, | |
| normalize_embeddings=True, | |
| show_progress_bar=False, | |
| convert_to_numpy=True, | |
| ).astype(_np.float32, copy=False) | |
| proba = _SOFTNESS_CLF["model"].predict_proba(enc)[:, 1] | |
| return {labels[i]: float(proba[i]) for i in range(len(labels))} | |
| def _normalise_skill_label(label: str) -> str: | |
| """ | |
| ESCO often disambiguates skills with parenthetical context, e.g. | |
| 'Python (programming language)'. Strip these so the resulting label | |
| keyword-matches whatever a JD's required_skills list contains. | |
| """ | |
| out = re.sub(r"\s*\([^)]+\)\s*", " ", label).strip() | |
| return re.sub(r"\s+", " ", out) | |
| def _chunk_resume_text(text: str, max_chunks: int = 40) -> list: | |
| """ | |
| Split a resume into short, embedding-friendly chunks. | |
| We DON'T pull in nltk just for this: a simple regex split on | |
| sentence-end punctuation + line breaks works fine for resume text, | |
| which is mostly bullets and short sentences. | |
| """ | |
| parts = re.split(r"(?<=[\.!?])\s+|\n+|•|·|■|►|▪|◦", text or "") | |
| chunks = [] | |
| for p in parts: | |
| s = p.strip() | |
| if 12 <= len(s) <= 280: | |
| chunks.append(s) | |
| if len(chunks) >= max_chunks: | |
| break | |
| return chunks | |
| def _esco_extract_skills(text: str, | |
| threshold: float = 0.55, | |
| top_k: int = 35) -> tuple[list, list]: | |
| """ | |
| Return (hard_skills, soft_skills) — ESCO labels semantically present in | |
| `text`, split by transversal-skill classification. | |
| Algorithm: | |
| 1. Split resume into ≤40 short chunks. | |
| 2. Encode chunks with JobBERT-v2 → (C, dim) L2-normalised matrix. | |
| 3. Dot-product with cached ESCO matrix → (C, N) cosine similarities. | |
| 4. For each skill, take the **max** similarity across chunks. | |
| 5. Keep top-k whose max-sim ≥ threshold; route by transversal flag. | |
| """ | |
| if not _ESCO_OK or not text: | |
| return [], [] | |
| chunks = _chunk_resume_text(text) | |
| if not chunks: | |
| return [], [] | |
| import numpy as _np | |
| q = _ESCO_MODEL.encode( | |
| chunks, | |
| normalize_embeddings=True, | |
| show_progress_bar=False, | |
| convert_to_numpy=True, | |
| ).astype(_np.float32, copy=False) | |
| sims = q @ _ESCO_EMBEDS.T # (chunks, N) | |
| per_skl = sims.max(axis=0) # best chunk-match per skill | |
| keep = _np.argsort(per_skl)[::-1][: top_k * 3] # over-pull, filter | |
| labels = _ESCO_META["labels"] | |
| hard: list[str] = [] | |
| soft: list[str] = [] | |
| seen: set[str] = set() | |
| for idx in keep: | |
| score = float(per_skl[idx]) | |
| if score < threshold: | |
| break # sorted desc → rest are below threshold | |
| label = _normalise_skill_label(labels[idx]) | |
| key = label.lower() | |
| if key in seen: | |
| continue | |
| seen.add(key) | |
| if int(idx) in _ESCO_SOFT_IDX: | |
| soft.append(label) | |
| else: | |
| hard.append(label) | |
| if len(hard) + len(soft) >= top_k: | |
| break | |
| return hard, soft | |
| # ============================================================================= | |
| # LOAD SKILLS TAXONOMY | |
| # ============================================================================= | |
| def _load_taxonomy() -> list: | |
| """ | |
| Load master skills list from data/skills_taxonomy.txt. | |
| Returns lowercase list. Falls back to a built-in starter list if | |
| the file doesn't exist yet. | |
| """ | |
| if os.path.exists(config.SKILLS_TAXONOMY_PATH): | |
| with open(config.SKILLS_TAXONOMY_PATH, "r", encoding="utf-8") as f: | |
| skills = [line.strip().lower() for line in f if line.strip()] | |
| return skills | |
| # Built-in starter taxonomy — covers most IT roles | |
| # Add more to data/skills_taxonomy.txt to extend this | |
| return [ | |
| # Languages | |
| "python", "java", "javascript", "typescript", "c++", "c#", "golang", | |
| "ruby", "php", "swift", "kotlin", "scala", "r", "matlab", "perl", | |
| # Web | |
| "react", "angular", "vue", "node.js", "django", "flask", "fastapi", | |
| "spring boot", "html", "css", "rest api", "graphql", | |
| # Data & ML | |
| "sql", "mysql", "postgresql", "mongodb", "redis", "elasticsearch", | |
| "pandas", "numpy", "scikit-learn", "tensorflow", "pytorch", "keras", | |
| "machine learning", "deep learning", "nlp", "computer vision", | |
| "data analysis", "data science", "statistics", "tableau", "power bi", | |
| "excel", "spark", "hadoop", "hive", | |
| # Cloud & DevOps | |
| "aws", "azure", "gcp", "docker", "kubernetes", "jenkins", "git", | |
| "github", "gitlab", "ci/cd", "terraform", "ansible", "linux", | |
| # Other Tech | |
| "agile", "scrum", "jira", "confluence", "microservices", "kafka", | |
| "airflow", "dbt", "snowflake", "databricks", | |
| # HR / Business (for non-tech JDs) | |
| "sap", "workday", "salesforce", "crm", "erp", "hris", | |
| ] | |
| _SKILLS_TAXONOMY = _load_taxonomy() | |
| # Soft skills vocabulary for pattern matching | |
| _SOFT_SKILLS = [ | |
| "communication", "leadership", "teamwork", "collaboration", "problem solving", | |
| "critical thinking", "time management", "adaptability", "creativity", | |
| "stakeholder management", "presentation", "negotiation", "mentoring", | |
| "conflict resolution", "decision making", "project management", | |
| "analytical", "detail oriented", "self motivated", "multitasking", | |
| ] | |
| # Degree keywords for education extraction | |
| _DEGREE_KEYWORDS = [ | |
| "b.tech", "b.e.", "b.sc", "bsc", "btech", "bachelor", "b.a", "ba", | |
| "m.tech", "m.e.", "m.sc", "msc", "mtech", "master", "m.a", "ma", "mba", | |
| "ph.d", "phd", "doctorate", "diploma", "pgdm", "pgpm", | |
| ] | |
| # ============================================================================= | |
| # 1. CONTACT INFORMATION | |
| # ============================================================================= | |
| # ── Indian city / state database for location fallback ─────────────────────── | |
| # Used when the LLM returns an empty location field. | |
| # Source: India Census 2011 + IT-hub common names. | |
| _INDIAN_CITIES = [ | |
| # Tier-1 metros | |
| "Mumbai", "Delhi", "Bangalore", "Bengaluru", "Hyderabad", "Chennai", | |
| "Kolkata", "Pune", "Ahmedabad", "Surat", | |
| # Tier-2 | |
| "Jaipur", "Lucknow", "Kanpur", "Nagpur", "Indore", "Thane", "Bhopal", | |
| "Visakhapatnam", "Vizag", "Patna", "Vadodara", "Ludhiana", "Agra", | |
| "Nashik", "Faridabad", "Meerut", "Rajkot", "Varanasi", "Prayagraj", | |
| "Allahabad", "Guwahati", "Dehradun", "Jammu", "Aurangabad", | |
| # NCR / satellite cities | |
| "Noida", "Gurgaon", "Gurugram", "Ghaziabad", "Greater Noida", "Navi Mumbai", | |
| # IT hubs & regional capitals | |
| "Chandigarh", "Coimbatore", "Kochi", "Cochin", "Trivandrum", | |
| "Thiruvananthapuram", "Mysore", "Mysuru", "Mangalore", "Mangaluru", | |
| "Hubballi", "Hubli", "Vijayawada", "Madurai", "Bhubaneswar", "Cuttack", | |
| "Ranchi", "Raipur", "Jabalpur", "Gwalior", "Jodhpur", "Udaipur", "Kota", | |
| "Amritsar", "Jalandhar", "Dhanbad", "Srinagar", | |
| ] | |
| _INDIAN_STATES = [ | |
| "Maharashtra", "Karnataka", "Tamil Nadu", "Andhra Pradesh", "Telangana", | |
| "Gujarat", "Rajasthan", "Uttar Pradesh", "West Bengal", "Madhya Pradesh", | |
| "Kerala", "Punjab", "Haryana", "Bihar", "Odisha", "Jharkhand", "Assam", | |
| "Delhi", "NCR", "NCT", | |
| ] | |
| def _extract_location_from_text(text: str) -> str: | |
| """ | |
| Best-effort location extraction from raw resume text. | |
| Strategy: | |
| 1. 'City, State' / 'City - State' pattern with known Indian states. | |
| 2. Known city name in the first ~350 chars (header zone). | |
| 3. Full-text scan up to 600 chars. | |
| Returns a city/location string, or "" if nothing found. | |
| """ | |
| # Pattern 1 — "City, State" with a recognised Indian state | |
| state_pat = '|'.join(re.escape(s) for s in _INDIAN_STATES) | |
| m = re.search( | |
| rf'([A-Z][a-zA-Z\s]{{2,25}})[,\s\-]+(?:{state_pat})\b', | |
| text | |
| ) | |
| if m: | |
| # Return the full match (e.g. "Noida, UP" or "Mumbai, Maharashtra") | |
| return m.group(0).strip().rstrip(',').strip() | |
| # Pattern 2 — header zone (first 350 chars) | |
| header = text[:350] | |
| for city in _INDIAN_CITIES: | |
| if re.search(r'\b' + re.escape(city) + r'\b', header, re.IGNORECASE): | |
| # Try to capture a trailing state/region on the same line | |
| city_m = re.search( | |
| r'\b' + re.escape(city) + r'\b[,\s]*([^\n]{0,30})', | |
| header, re.IGNORECASE | |
| ) | |
| if city_m: | |
| extra = city_m.group(1).strip().rstrip(',').strip() | |
| extra_first_line = extra.split('\n')[0].strip() | |
| if any(s.lower() in extra_first_line.lower() for s in _INDIAN_STATES): | |
| return f"{city.title()}, {extra_first_line}" | |
| return city.title() | |
| # Pattern 3 — full text scan (slower, last resort) | |
| for city in _INDIAN_CITIES: | |
| if re.search(r'\b' + re.escape(city) + r'\b', text[:600], re.IGNORECASE): | |
| return city.title() | |
| return "" | |
| # ── Post-processing scrubs for LLM hallucinations ──────────────────────────── | |
| # Built once at module load — cheap lower-case lookups. | |
| _LOCATION_TOKENS = ( | |
| {c.lower() for c in _INDIAN_CITIES} | |
| | {s.lower() for s in _INDIAN_STATES} | |
| | {"ncr", "india", "region", "area", "north", "south", "east", "west"} | |
| ) | |
| # Pure business-function / department / section-header tokens. | |
| # A "company" composed ONLY of these is a section header, not a real employer. | |
| _FUNCTION_TOKENS = { | |
| # Departments / functions | |
| "operations", "management", "marketing", "sales", "finance", "accounting", | |
| "administration", "administrative", "hr", "recruitment", "talent", | |
| "business", "analysis", "analytics", "strategy", "development", "planning", | |
| "credit", "risk", "compliance", "audit", "treasury", "regulatory", | |
| "support", "services", "service", "customer", "client", "relationship", | |
| "insurance", "banking", "investment", "wealth", "private", "retail", | |
| "corporate", "commercial", "consumer", "product", "project", "program", | |
| "delivery", "engineering", "production", "manufacturing", "supply", "chain", | |
| "logistics", "procurement", "quality", "assurance", "control", "controls", | |
| "technology", "information", "data", "research", | |
| "transformation", "digital", "general", "cash", "automation", "platform", | |
| "process", "processes", "digitization", "fraud", "financial", | |
| "leadership", "leadership", "revenue", "pricing", "approval", "approvals", | |
| "review", "reviewed", "approval", "growth", "performance", | |
| # Industries / domains often mistaken for companies | |
| "pension", "life", "health", "asset", "lending", "loans", "mortgage", | |
| "payments", "fintech", "ecommerce", "e-commerce", | |
| # Acronyms & metrics frequently mis-extracted as company names | |
| "kpi", "kpis", "api", "apis", "roi", "sla", "kra", "kras", | |
| "bpo", "kpo", "p&l", "pnl", "ai", "ml", "saas", | |
| # UI/UX/IT abbreviations sometimes returned as "company" | |
| "ui", "ux", "it", | |
| # Country / region tokens (frequently leaked from section headers like "UK Life") | |
| "uk", "us", "usa", "uae", "europe", "asia", "apac", "emea", "global", | |
| # Degree leaks | |
| "b.tech", "btech", "m.tech", "mtech", "b.sc", "bsc", "m.sc", "msc", | |
| "mba", "ba", "ma", "phd", "pgdm", | |
| # Resume section-header words (LLM sometimes returns these as "companies") | |
| "profile", "summary", "core", "competencies", "competency", | |
| "professional", "work", "history", "objective", "overview", | |
| "highlights", "expertise", "achievements", "awards", | |
| "education", "documentation", "contact", "address", "id", "email", | |
| # Product / industry descriptor words (not a company name on their own) | |
| "personal", "durables", "consumer", "sme", "msme", | |
| # Finance / compliance acronyms often leaked from job descriptions | |
| "kyc", "aml", "emi", "npa", "nro", "nre", "upi", "pos", | |
| "nbfc", "mfi", "rbi", "irda", "amfi", "sebi", "lap", "tprm", | |
| # Banking / NBFC function words commonly leaked as fake company names | |
| "account", "accounts", "key", "vendor", "payment", "payments", | |
| "escrow", "nodal", "loan", "loans", "detection", "collection", | |
| "collections", "people", "stakeholder", "stakeholders", | |
| "limit", "loading", "borrower", "disbursement", "reconciliation", | |
| "portfolio", "bucket", "delinquency", "finops", "fin-ops", | |
| "housing", "home", "projects", "sourcing", "monitoring", | |
| "team", "building", "performance", "legal", "linkedin", | |
| "inr", "usd", "pre", "paid", "instrument", "instruments", | |
| # Job-title seniority/level words — appear in role prefixes, not company names | |
| "head", "vice", "vp", "avp", "svp", "evp", | |
| } | |
| # Job-title prefixes that the LLM sometimes pastes in front of a company name | |
| # e.g. "Country Head – League of Extraordinary" → starts with "Country Head" | |
| # Any company-string that starts with one of these prefixes is a role label, not a company. | |
| _JOB_TITLE_PREFIX_RE = re.compile( | |
| r'^(?:' | |
| r'country\s+head|head\s+of|vp\s+of|vice\s+president(?:\s+of)?|' | |
| r'avp\s+(?:of\s+)?|svp\s+(?:of\s+)?|evp\s+(?:of\s+)?|' | |
| r'chief\s+(?:executive|operating|financial|technology|marketing|people)|' | |
| r'managing\s+director|general\s+manager|deputy\s+(?:general\s+)?manager|' | |
| r'senior\s+vice\s+president|assistant\s+vice\s+president|' | |
| r'regional\s+(?:head|manager|director)|national\s+(?:head|manager|director)|' | |
| r'zonal\s+(?:head|manager|director)|business\s+head|' | |
| r'president\s+(?:and|&)\s+|founder\s+(?:and|&)\s+|' | |
| r'cluster\s+head|branch\s+head|area\s+(?:head|manager)' | |
| r')\b', | |
| re.IGNORECASE, | |
| ) | |
| # Corporate suffixes — any of these in the string strongly suggests a real company | |
| _CORPORATE_SUFFIXES = { | |
| "ltd", "ltd.", "limited", "pvt", "pvt.", "private", | |
| "inc", "inc.", "incorporated", "corp", "corp.", "corporation", | |
| "llc", "llp", "plc", "gmbh", "ag", "sa", "co", "co.", | |
| "company", "bank", "industries", "technologies", "solutions", | |
| "systems", "services", "consulting", "consultants", "enterprises", | |
| "group", "holdings", "international", | |
| } | |
| def _looks_like_location(text: str) -> bool: | |
| """ | |
| True iff `text` is essentially a place name (no real company-name signal). | |
| Used to reject hallucinated 'last_company' values like 'Delhi-NCR'. | |
| """ | |
| if not text: | |
| return False | |
| t = text.lower().strip(' .,-–—()') | |
| if not t: | |
| return False | |
| tokens = [w for w in re.split(r"[\s,\-–—/&]+", t) if w] | |
| if not tokens: | |
| return False | |
| connectors = {"and", "the", "of", "in", "at"} | |
| return all((tok in _LOCATION_TOKENS or tok in connectors) for tok in tokens) | |
| def _looks_like_function(text: str) -> bool: | |
| """ | |
| True iff `text` is purely a job/department/function name with no company- | |
| specific proper-noun tokens. E.g. 'Insurance Operations', 'Credit Management', | |
| 'Bank', 'FCA & PRA Compliance'. | |
| Decision rule: a real company must have at least one PROPER NOUN — i.e. a | |
| token that is NOT in any of (function, location, regulator, connector, | |
| corporate-suffix) sets. Otherwise it's just a description. | |
| """ | |
| if not text: | |
| return False | |
| t = re.sub(r"\s+", " ", text.lower()).strip() | |
| if not t: | |
| return False | |
| tokens = [w.strip(".,") for w in re.split(r"[\s/&\-–—,]+", t) if w] | |
| tokens = [tok for tok in tokens if tok] | |
| if not tokens: | |
| return False | |
| connectors = {"and", "the", "of", "in", "at", "for"} | |
| # Regulator / non-company acronyms (UK Financial Conduct Authority, etc.) | |
| regulators = { | |
| "fca", "pra", "sec", "rbi", "icai", "fda", "hipaa", "sox", | |
| "gdpr", "pci", "irda", "sebi", "nse", "bse", | |
| } | |
| # A proper-noun token = not in any of the "common / generic" pools | |
| proper_noun_tokens = [ | |
| tok for tok in tokens | |
| if tok not in _FUNCTION_TOKENS | |
| and tok not in _LOCATION_TOKENS | |
| and tok not in _CORPORATE_SUFFIXES | |
| and tok not in connectors | |
| and tok not in regulators | |
| ] | |
| # No proper noun → it's just function/location/suffix words → reject | |
| if not proper_noun_tokens: | |
| return True | |
| return False | |
| def _looks_like_garbage(text: str) -> bool: | |
| """ | |
| True iff `text` is obviously not a company name — bullet text, all | |
| punctuation, too short, oversized fragments, or PDF unicode artifacts. | |
| """ | |
| if not text: | |
| return True | |
| t = text.strip() | |
| # Contains a newline → multi-line section fragment, not a single company | |
| if '\n' in t or '\r' in t: | |
| return True | |
| # Absurdly long → entire section header / bullet cluster | |
| if len(t) > 80: | |
| return True | |
| # Starts with a bullet/symbol or unicode dash → bullet text fragment | |
| # Includes ASCII hyphen, em-dash (—), en-dash (–), minus (−) | |
| if re.match(r'^[•●○◦►▪▫◆◇■□★☆\-\*\+§¶—–−]', t): | |
| return True | |
| # PDF unicode private-use area artifacts (Wingdings glyphs etc.) | |
| # These appear as , , … when PDF bullets are extracted | |
| if re.match(r'^[-]', t): | |
| return True | |
| # Anywhere in the first 3 chars | |
| if any(0xf000 <= ord(ch) <= 0xf0ff for ch in t[:3]): | |
| return True | |
| # Has no alphabetic content at all | |
| if not re.search(r'[A-Za-z]', t): | |
| return True | |
| # Strip non-letter chars; if ≤ 2 letters remain → garbage | |
| # (catches "UI", "P&L", "HR", "IT" etc.) | |
| letters_only = re.sub(r'[^A-Za-z]', '', t) | |
| if len(letters_only) <= 2: | |
| return True | |
| # Contains internal bullet markers → multi-bullet fragment | |
| if re.search(r'[•●◦►▪▫◆◇■□]', t): | |
| return True | |
| # Unbalanced parentheses → truncated/corrupted string | |
| # e.g. "Anti, Money Laundering (AML" or "FIU IND (CIMS & DAKSH" | |
| if t.count('(') != t.count(')'): | |
| return True | |
| # Contains '@' → email address leaked as company name | |
| if '@' in t: | |
| return True | |
| # Starts with a job-title prefix → role label, not a company name | |
| # e.g. "Country Head League of Extraordinary", "Head of Insurance" | |
| if _JOB_TITLE_PREFIX_RE.match(t): | |
| return True | |
| # Suspicious starts — verb or structural phrase that identifies a non-company string | |
| suspicious_starts = ( | |
| 'executed ', 'facilitated ', 'currently', 'working with ', 'handled ', 'guided ', 'collaborated ', | |
| 'monitored ', 'led ', 'managed ', 'developed ', 'created ', | |
| 'designed ', 'built ', 'implemented ', 'delivered ', 'achieved ', | |
| 'worked ', 'responsible ', 'role ', 'duty ', | |
| 'reviewed ', 'approved ', 'drove ', 'utilize ', 'utilized ', | |
| 'utilise ', 'utilised ', 'directed ', 'oversaw ', 'spearheaded ', | |
| 'ensured ', 'supported ', 'assisted ', 'coordinated ', | |
| 'analyzed ', 'analysed ', | |
| ) | |
| if t.lower().startswith(suspicious_starts): | |
| return True | |
| return False | |
| def _clean_company_name(co: str) -> str: | |
| """ | |
| Best-effort cleanup of an LLM-emitted company name. | |
| Rejects: | |
| - Bullet text fragments ("• Executed daily reviews") | |
| - Pure locations ("Delhi NCR") | |
| - Pure job functions ("Insurance Operations", "Credit Management") | |
| - Punctuation-only or too-short strings | |
| Keeps: | |
| - 'Infosys' → 'Infosys' | |
| - 'HDFC Bank, Mumbai' → 'HDFC Bank' | |
| - 'Groz Engineering Tools Pvt. Ltd.' → 'Groz Engineering Tools Pvt. Ltd.' | |
| - 'Yes Bank Ltd' → 'Yes Bank Ltd' | |
| """ | |
| if not co: | |
| return "" | |
| raw = co.strip() | |
| if _looks_like_garbage(raw): return "" | |
| if _looks_like_location(raw): return "" | |
| if _looks_like_function(raw): return "" | |
| # Strip trailing location markers via segment split | |
| parts = re.split(r"\s*[,\-–—]\s*", raw) | |
| kept = [] | |
| for p in parts: | |
| p = p.strip() | |
| if not p: | |
| continue | |
| if _looks_like_location(p): | |
| continue | |
| if _looks_like_function(p): | |
| continue | |
| kept.append(p) | |
| cleaned = ", ".join(kept) if kept else "" | |
| # Final sanity check after splitting | |
| if not cleaned or _looks_like_garbage(cleaned): | |
| return "" | |
| return cleaned.strip() | |
| def _filter_project_entries(projects: list, companies: list) -> list: | |
| """ | |
| Remove entries that look like company names or pure work-history bullets | |
| from the projects list. The LLM sometimes leaks employer names here. | |
| """ | |
| if not projects: | |
| return [] | |
| co_keys = {c.lower().strip() for c in companies if c} | |
| cleaned = [] | |
| for p in projects: | |
| if not p or not p.strip(): | |
| continue | |
| ps = p.strip() | |
| # Drop direct company-name matches | |
| if ps.lower() in co_keys: | |
| continue | |
| # Drop bare location strings | |
| if _looks_like_location(ps): | |
| continue | |
| # Drop entries whose leading token is a known company name. | |
| # Splits on | — – - , ( so "HDFC Bank, Mumbai (2018-2022)" → "HDFC Bank" | |
| first_part = re.split(r'[|—–\-,(]', ps, 1)[0].strip() | |
| if first_part and first_part.lower() in co_keys: | |
| continue | |
| # Drop entries whose first two words match a company prefix | |
| # e.g. "HDFC Bank quarterly review" when company is "HDFC Bank" | |
| first_two = " ".join(first_part.split()[:2]).lower() | |
| if first_two and first_two in co_keys: | |
| continue | |
| cleaned.append(ps) | |
| return cleaned | |
| def _years_from_tenures(tenures: list) -> float: | |
| """ | |
| Sum non-overlapping calendar time across a list of company_tenures | |
| entries. "Present"/"Current"/"Now" are anchored to today's date — NOT | |
| the date the CV was written. Returns total years (rounded to 1 dp), or | |
| 0 if no parseable date ranges were found. | |
| Accepts both formats: | |
| - "Company | Role | Start – End" (new format) | |
| - "Company – City (Start – End)" (legacy format) | |
| """ | |
| if not tenures: | |
| return 0.0 | |
| _month = ( | |
| r'(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|' | |
| r'Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|' | |
| r'Nov(?:ember)?|Dec(?:ember)?)' | |
| ) | |
| range_re = re.compile( | |
| rf'({_month}\s*\'?\d{{2,4}}|20\d{{2}})\s*[-–—]\s*' | |
| rf'({_month}\s*\'?\d{{2,4}}|20\d{{2}}|[Pp]resent|[Cc]urrent|[Nn]ow|[Oo]ngoing)', | |
| re.IGNORECASE, | |
| ) | |
| now = datetime.now() | |
| pairs: list = [] | |
| for t in tenures: | |
| m = range_re.search(str(t)) | |
| if not m: | |
| continue | |
| start_str, end_str = m.group(1).strip(), m.group(2).strip().lower() | |
| start = _parse_date_safe(start_str) | |
| end = now if end_str in ('present', 'current', 'now', 'ongoing') \ | |
| else _parse_date_safe(end_str) | |
| if start and end and end >= start: | |
| pairs.append((start, end)) | |
| if not pairs: | |
| return 0.0 | |
| merged_months = 0 | |
| for s, e in _merge_date_ranges(pairs): | |
| d = relativedelta(e, s) | |
| merged_months += d.years * 12 + d.months | |
| return round(merged_months / 12, 1) | |
| def _build_tenures_from_text(text: str, companies: list) -> list: | |
| """ | |
| Regex fallback: build "Company – City (Month Year – Month Year)" entries | |
| by scanning the raw resume text for date ranges near each known company name. | |
| Called only when the LLM returns an empty company_tenures list. | |
| Strategy per company: | |
| 1. Find the company name's first occurrence in the text. | |
| 2. In the next ~400 chars search for a month-year date range. | |
| 3. Optionally capture a known Indian city in the same window. | |
| 4. Emit the formatted tenure string. | |
| """ | |
| if not companies: | |
| return [] | |
| # Pattern A — "Month Year – Month Year" (most common in structured CVs) | |
| _month = ( | |
| r'(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|' | |
| r'Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|' | |
| r'Nov(?:ember)?|Dec(?:ember)?)' | |
| ) | |
| _date_range_re = re.compile( | |
| rf'({_month}\s*\'?\d{{2,4}})\s*[-–—]\s*' | |
| rf'({_month}\s*\'?\d{{2,4}}|[Pp]resent|[Cc]urrent)', | |
| re.IGNORECASE, | |
| ) | |
| # Pattern B — "Year – Year" or "Year – Present" (year-only ranges) | |
| _year_range_re = re.compile( | |
| r'(20\d{2})\s*[-–—]\s*(20\d{2}|[Pp]resent|[Cc]urrent)', | |
| re.IGNORECASE, | |
| ) | |
| tenures: list = [] | |
| seen: set = set() | |
| for company in companies: | |
| # Find position of this company in the full text (case-insensitive) | |
| try: | |
| idx = text.lower().index(company.lower()) | |
| except ValueError: | |
| continue | |
| # 400-char window starting at the company mention | |
| window = text[idx: idx + 400] | |
| m_month = _date_range_re.search(window) | |
| m_year = _year_range_re.search(window) | |
| if m_month and m_year: | |
| # Take whichever date range appears first (closest to the company) | |
| if m_month.start() <= m_year.start(): | |
| start_str = m_month.group(1).strip() | |
| end_raw = m_month.group(2).strip() | |
| else: | |
| start_str = m_year.group(1).strip() | |
| end_raw = m_year.group(2).strip() | |
| elif m_month: | |
| start_str = m_month.group(1).strip() | |
| end_raw = m_month.group(2).strip() | |
| elif m_year: | |
| start_str = m_year.group(1).strip() | |
| end_raw = m_year.group(2).strip() | |
| else: | |
| continue | |
| # Normalise "present"/"current" → "Present" | |
| end_str = "Present" if end_raw.lower() in ("present", "current") else end_raw | |
| # Look for a known Indian city in the first 200 chars of the window | |
| city = "" | |
| for cname in _INDIAN_CITIES: | |
| if re.search(r'\b' + re.escape(cname) + r'\b', window[:200], re.IGNORECASE): | |
| city = cname.title() | |
| break | |
| dates = f"{start_str} – {end_str}" | |
| key = (company.lower(), dates.lower()) | |
| if key in seen: | |
| continue | |
| seen.add(key) | |
| if city: | |
| tenures.append(f"{company} – {city} ({dates})") | |
| else: | |
| tenures.append(f"{company} ({dates})") | |
| return tenures | |
| _NAME_SKIP_WORDS = { | |
| # Document type / section labels that bubble up into the resume header | |
| 'resume', 'cv', 'curriculum', 'vitae', 'profile', 'objective', | |
| 'summary', 'contact', 'information', 'details', 'education', | |
| 'experience', 'skills', 'certifications', 'projects', 'hobbies', | |
| 'interests', 'achievements', 'career', 'personal', 'professional', | |
| # Job-title nouns — common false positives when scanning header lines | |
| 'senior', 'junior', 'lead', 'manager', 'analyst', 'engineer', | |
| 'developer', 'consultant', 'specialist', 'executive', 'director', | |
| 'intern', 'trainee', 'associate', 'accountant', 'designer', | |
| 'copywriter', 'writer', 'marketer', 'page', 'of', | |
| # Address / contact noise | |
| 'mobile', 'phone', 'email', 'address', 'linkedin', 'github', | |
| 'india', 'delhi', 'mumbai', 'bangalore', 'pune', 'hyderabad', | |
| } | |
| # Explicit labels that prefix a name on the resume header | |
| _NAME_LABEL_PATTERNS = [ | |
| # "Name: John Doe" / "Candidate Name - John Doe" | |
| re.compile( | |
| r'^\s*(?:name|candidate\s*name|full\s*name|applicant\s*name)\s*[:\-–]\s*' | |
| r'([A-Za-z][A-Za-z\.\-\']+(?:\s+[A-Za-z][A-Za-z\.\-\']+){1,3})\s*$', | |
| re.IGNORECASE, | |
| ), | |
| # "I am John Doe" / "My name is John Doe" | |
| re.compile( | |
| r'^\s*(?:i\s+am|my\s+name\s+is|this\s+is)\s+' | |
| r'([A-Za-z][A-Za-z\.\-\']+(?:\s+[A-Za-z][A-Za-z\.\-\']+){1,3})\b', | |
| re.IGNORECASE, | |
| ), | |
| ] | |
| # 2–4 ALL-CAPS tokens, optionally separated by " - " — very common header style | |
| _NAME_ALLCAPS_PATTERN = re.compile( | |
| r'^\s*([A-Z][A-Z\.\-\']{1,}(?:\s+[A-Z][A-Z\.\-\']{1,}){1,3})\s*$' | |
| ) | |
| # Title-cased "First Last [Middle]" pattern, allowing single-letter middles | |
| _NAME_TITLECASE_PATTERN = re.compile( | |
| r'^\s*([A-Z][a-z]+(?:\s+[A-Z]\.?)?\s+[A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)\s*$' | |
| ) | |
| def _looks_like_name(text: str) -> bool: | |
| """Lightweight gate used by every regex/NER branch.""" | |
| words = text.strip().split() | |
| if not (2 <= len(words) <= 4): | |
| return False | |
| if re.search(r'[0-9@|/\\:_]', text): | |
| return False | |
| if any(w.lower() in _NAME_SKIP_WORDS for w in words): | |
| return False | |
| if len(text.strip()) < 4: | |
| return False | |
| return True | |
| def _header_lines(text: str, n: int = 10) -> list: | |
| """ | |
| Return the first `n` non-empty lines of the resume. Name extraction is | |
| restricted to this slice because resume names live in the header. | |
| """ | |
| out = [] | |
| for ln in text.split('\n'): | |
| s = ln.strip() | |
| if not s: | |
| continue | |
| out.append(s) | |
| if len(out) >= n: | |
| break | |
| return out | |
| def extract_candidate_name(text: str, file_name: str = "") -> str: | |
| """ | |
| Extract candidate name from a resume. | |
| Strategy (in order of reliability): | |
| 1. spaCy PERSON entity — but only on the **first 10 non-empty lines**, | |
| not the full document. Stops mid-resume "John, my mentor" hits. | |
| 2. Regex fallback — explicit "Name: John Doe" labels. | |
| 3. Regex fallback — ALL-CAPS header (e.g. "GAURAV JOSHI"). | |
| 4. Regex fallback — Title-case "First Last" pattern. | |
| 5. Loose line scan inside the header zone. | |
| 6. Filename derivation as a last resort. | |
| Args: | |
| text : Raw resume text | |
| file_name : Original filename — used as last-resort fallback | |
| Returns: | |
| Candidate name string, or "Unknown" if nothing found. | |
| """ | |
| if not text: | |
| return _name_from_filename(file_name) | |
| header = _header_lines(text, n=10) | |
| header_text = "\n".join(header) | |
| # ── Strategy 1: spaCy NER on header zone only ──────────────────────────── | |
| if _SPACY_OK and _NLP and header_text: | |
| try: | |
| doc = _NLP(header_text) | |
| for ent in doc.ents: | |
| if ent.label_ == "PERSON" and _looks_like_name(ent.text): | |
| cleaned = re.sub(r"[^a-zA-Z\s\.\-']", '', ent.text).strip() | |
| if cleaned and len(cleaned) >= 4: | |
| return cleaned.title() | |
| except Exception: | |
| pass # spaCy occasionally chokes on garbled OCR text | |
| # ── Strategy 2: Regex — explicit "Name:" labels in the header ──────────── | |
| for ln in header: | |
| for pat in _NAME_LABEL_PATTERNS: | |
| m = pat.search(ln) | |
| if m and _looks_like_name(m.group(1)): | |
| return m.group(1).strip().title() | |
| # ── Strategy 3: Regex — ALL-CAPS header line (top 5 lines only) ────────── | |
| for ln in header[:5]: | |
| m = _NAME_ALLCAPS_PATTERN.match(ln) | |
| if m and _looks_like_name(m.group(1)): | |
| return m.group(1).strip().title() | |
| # ── Strategy 4: Regex — Title-cased "First Last" ───────────────────────── | |
| for ln in header[:6]: | |
| m = _NAME_TITLECASE_PATTERN.match(ln) | |
| if m and _looks_like_name(m.group(1)): | |
| return m.group(1).strip().title() | |
| # ── Strategy 5: Loose pattern scan over the full header ────────────────── | |
| for ln in header: | |
| words = ln.split() | |
| if (2 <= len(words) <= 4 and | |
| not re.search(r'[0-9@|/\\:_\.]', ln) and | |
| all(w[0].isupper() for w in words if w) and | |
| not any(w.lower() in _NAME_SKIP_WORDS for w in words) and | |
| len(ln) >= 5): | |
| return ln.title() | |
| # ── Strategy 6: Derive from filename ───────────────────────────────────── | |
| return _name_from_filename(file_name) | |
| def _name_from_filename(file_name: str) -> str: | |
| """ | |
| Derive a display name from a resume filename. | |
| Examples: | |
| "priya_sharma.pdf" → "Priya Sharma" | |
| "Gaurav_Joshi_Resume pdf.pdf" → "Gaurav Joshi" | |
| "Naukri_AayushiShukla[6y_0m].pdf" → "Aayushishukla" | |
| "kartik CV.pdf" → "Kartik" | |
| "resume_1.pdf" → "Unknown" | |
| """ | |
| if not file_name: | |
| return "Unknown" | |
| stem = os.path.splitext(os.path.basename(file_name))[0] | |
| # Replace underscores, hyphens, dots with spaces | |
| stem = re.sub(r'[_\-\.]+', ' ', stem).strip() | |
| # Strip brackets and their contents e.g. [6y_0m] | |
| stem = re.sub(r'\[.*?\]|\(.*?\)', ' ', stem).strip() | |
| # Remove any remaining non-alphabetic characters from each token | |
| words = [re.sub(r'[^a-zA-Z]', '', w) for w in stem.split()] | |
| words = [w for w in words if w] # drop empty tokens after stripping | |
| # Skip generic filename tokens (including file-extension leftovers) | |
| generic = { | |
| 'resume', 'cv', 'curriculum', 'vitae', 'application', | |
| 'candidate', 'updated', 'final', 'new', 'f', 'v', 'v2', 'v3', | |
| 'pdf', 'docx', 'doc', 'txt', | |
| 'naukri', 'linkedin', 'profile', 'my', 'latest', 'copy', | |
| '1', '2', '3', '2024', '2025', '2026', | |
| } | |
| filtered = [w for w in words | |
| if w.lower() not in generic and len(w) >= 2 and not w.isdigit()] | |
| if len(filtered) >= 2: | |
| return ' '.join(w.title() for w in filtered[:3]) | |
| # Accept single meaningful word as a first-name-only fallback | |
| if len(filtered) == 1 and len(filtered[0]) >= 3: | |
| return filtered[0].title() | |
| return "Unknown" | |
| def extract_contact(text: str) -> dict: | |
| """ | |
| Extract email address and phone number from resume text. | |
| Phone extraction — multi-strategy, Indian-mobile-first: | |
| 1. +91 prefix → capture the 10 digits that follow (+91 removed as prefix) | |
| 2. 91 without + → same | |
| 3. Leading 0 → STD-style (0 removed as prefix) | |
| 4. Bare 10-digit mobile starting with 6-9 | |
| 5. Generic international fallback | |
| The old ``lstrip('+91')`` was a character-set strip (stripped any of | |
| '+','9','1'), which turned '+91 9953668257' → '53668257'. | |
| This version uses regex prefix removal so every digit is preserved. | |
| Returns: | |
| {"email": "x@y.com", "phone": "9876543210"} | |
| Values are empty string if not found. | |
| """ | |
| # Email — standard pattern | |
| email_match = re.search( | |
| r'[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}', text | |
| ) | |
| email = email_match.group(0) if email_match else "" | |
| phone = "" | |
| # Strategy 1: +91 [sep?] 10-digit-mobile (with optional mid-separators) | |
| m = re.search(r'\+91[\s\-\.]?([6-9]\d{2})[\s\-\.]?(\d{3})[\s\-\.]?(\d{4})', text) | |
| if m: | |
| phone = m.group(1) + m.group(2) + m.group(3) | |
| # Strategy 2: 91 (no +) then 10-digit mobile — e.g. "919876543210" | |
| if not phone: | |
| m = re.search(r'(?<!\d)91([6-9]\d{2})[\s\-\.]?(\d{3})[\s\-\.]?(\d{4})(?!\d)', text) | |
| if m: | |
| phone = m.group(1) + m.group(2) + m.group(3) | |
| # Strategy 3: Leading-0 STD format — e.g. "09876543210" | |
| if not phone: | |
| m = re.search(r'(?<!\d)0([6-9]\d{2})[\s\-\.]?(\d{3})[\s\-\.]?(\d{4})(?!\d)', text) | |
| if m: | |
| phone = m.group(1) + m.group(2) + m.group(3) | |
| # Strategy 4: Bare 10-digit Indian mobile (most common in Indian CVs) | |
| if not phone: | |
| m = re.search(r'(?<!\d)([6-9]\d{2})[\s\-\.]?(\d{3})[\s\-\.]?(\d{4})(?!\d)', text) | |
| if m: | |
| phone = m.group(1) + m.group(2) + m.group(3) | |
| # Strategy 5: Generic international number ≥ 10 digits | |
| if not phone: | |
| m = re.search(r'\+\d[\d\s\-\.]{9,14}\d', text) | |
| if m: | |
| digits = re.sub(r'\D', '', m.group(0)) | |
| # Remove +91 country code if present (regex prefix, not char-set) | |
| digits = re.sub(r'^91', '', digits) if len(digits) > 10 else digits | |
| phone = digits[-10:] if len(digits) >= 10 else digits | |
| return {"email": email, "phone": phone} | |
| def extract_urls(text: str) -> dict: | |
| """ | |
| Pull every clickable URL out of the resume and bucket them by type: | |
| - linkedin_url : LinkedIn profile (linkedin.com/in/…) | |
| - github_url : GitHub profile or repo (github.com/…) | |
| - portfolio_url : First non-LinkedIn / non-GitHub personal site | |
| (heuristic: looks like a portfolio domain or | |
| contains personal keywords) | |
| - other_urls : Every remaining URL — kept as a JSON list so the UI | |
| can render them as extra link chips. | |
| The patterns are loose on protocol — we accept "https://", "http://" or | |
| bare "www.linkedin.com/in/foo" because Indian CVs frequently strip the | |
| protocol when exporting from Word to PDF. | |
| Returns dict with all four keys; values are "" or [] when not found. | |
| """ | |
| # Canonical URL pattern — accepts protocol-less domains too. | |
| url_re = re.compile( | |
| r'(?:https?://|www\.)[A-Za-z0-9._~:/?#@!$&\'()*+,;=\-%]+', | |
| re.IGNORECASE, | |
| ) | |
| candidates = [] | |
| for m in url_re.finditer(text or ""): | |
| u = m.group(0).strip().rstrip('.,;:)]}') | |
| if u.lower().startswith("www."): | |
| u = "https://" + u | |
| if u not in candidates: | |
| candidates.append(u) | |
| # Also pick up bare LinkedIn slugs even when the user wrote just | |
| # "linkedin.com/in/foo" without www/https — common in scraped PDFs. | |
| for m in re.finditer( | |
| r'(?:^|\s)((?:linkedin\.com|github\.com)/[A-Za-z0-9._~:/?#@!$&\'()*+,;=\-%]+)', | |
| text or "", re.IGNORECASE, | |
| ): | |
| u = "https://" + m.group(1).strip().rstrip('.,;:)]}') | |
| if u not in candidates: | |
| candidates.append(u) | |
| linkedin = "" | |
| github = "" | |
| portfolio = "" | |
| others = [] | |
| PORTFOLIO_HINT_RE = re.compile( | |
| r'(portfolio|behance|dribbble|medium|substack|notion\.site|' | |
| r'github\.io|wordpress|wixsite|webflow|carrd|vercel\.app|' | |
| r'netlify\.app|pages\.dev)', | |
| re.IGNORECASE, | |
| ) | |
| SKIP_RE = re.compile( | |
| r'(google\.com/maps|wa\.me|tel:|mailto:|youtu\.be|youtube\.com/watch)', | |
| re.IGNORECASE, | |
| ) | |
| for u in candidates: | |
| lo = u.lower() | |
| if SKIP_RE.search(lo): | |
| continue | |
| if not linkedin and "linkedin.com" in lo: | |
| linkedin = u | |
| continue | |
| if not github and "github.com" in lo: | |
| github = u | |
| continue | |
| if not portfolio and PORTFOLIO_HINT_RE.search(lo): | |
| portfolio = u | |
| continue | |
| others.append(u) | |
| return { | |
| "linkedin_url" : linkedin, | |
| "github_url" : github, | |
| "portfolio_url" : portfolio, | |
| "other_urls" : others[:5], # cap at 5 to keep CSV row sane | |
| } | |
| # ============================================================================= | |
| # 2. SKILLS EXTRACTION | |
| # ============================================================================= | |
| def extract_skills(text: str, | |
| taxonomy: list = None, | |
| category: str = "", | |
| job_title: str = "", | |
| esco_threshold: float = 0.55, | |
| esco_top_k: int = 35) -> list: | |
| """ | |
| Extract technical skills from resume / JD text. | |
| Layered strategy (each layer adds to a single deduplicated set): | |
| 1. ESCO + JobBERT-v2 embedding lookup (Layer 1, recall-oriented). | |
| Built via build_esco_index.py. Catches semantic mentions like | |
| "built ETL pipelines" → Apache Airflow / Data Engineering even | |
| when the literal skill name never appears. | |
| 2. Taxonomy keyword match (precision-oriented). | |
| Anchors high-confidence exact matches against | |
| data/skills_taxonomy.txt — guarantees obvious skills like | |
| "Python" or "SQL" never get missed even if ESCO is unavailable. | |
| 3. Legacy TF-IDF classifier (train_skill_model.py). | |
| Only used when the ESCO index has NOT been built yet, so older | |
| deployments keep working until they run build_esco_index.py. | |
| Args: | |
| text : Raw resume or JD text | |
| taxonomy : Override skills list (defaults to loaded taxonomy) | |
| category : Job category hint for the legacy ML model | |
| job_title : Job title hint for the legacy ML model | |
| esco_threshold: Min cosine similarity to keep an ESCO label (0–1) | |
| esco_top_k : Max ESCO labels per call | |
| Returns: | |
| Sorted deduplicated list of skill strings. | |
| """ | |
| if not text: | |
| return [] | |
| # Soft-skill block set. Any label whose lowercase form is in here will be | |
| # rejected by _add() regardless of which layer produced it — this is what | |
| # prevents "Leadership"/"Communication" from leaking out of the taxonomy | |
| # match and into technical_skills. | |
| _soft_blocklist = {s.lower() for s in _SOFT_SKILLS} | |
| if _ESCO_OK and _ESCO_META: | |
| _labels = _ESCO_META.get("labels", []) | |
| for _idx in _ESCO_SOFT_IDX: | |
| if 0 <= _idx < len(_labels): | |
| _soft_blocklist.add(_normalise_skill_label(_labels[_idx]).lower()) | |
| found = {} # lower-case key → display label (preserves first form seen) | |
| def _add(label: str) -> None: | |
| if not label: | |
| return | |
| key = label.lower() | |
| if key in _soft_blocklist: | |
| return | |
| if key not in found: | |
| found[key] = label | |
| # ── Layer 1: ESCO semantic lookup (hard skills only — soft hits are | |
| # routed by transversal flag and consumed by extract_soft_skills) ───────── | |
| if _ESCO_OK: | |
| hard_hits, _soft_hits = _esco_extract_skills(text, | |
| threshold=esco_threshold, | |
| top_k=esco_top_k) | |
| for s in hard_hits: | |
| _add(s) | |
| # ── Layer 2: Taxonomy keyword match (precision floor) ──────────────────── | |
| tax = taxonomy or _SKILLS_TAXONOMY | |
| text_lower = text.lower() | |
| for skill in tax: | |
| pattern = r'\b' + re.escape(skill) + r'\b' | |
| if re.search(pattern, text_lower): | |
| display = skill.upper() if len(skill) <= 3 else skill.title() | |
| _add(display) | |
| # ── Layer 3: Legacy TF-IDF predictions (only if ESCO not yet built) ────── | |
| if not _ESCO_OK and _ML_MODEL_OK: | |
| for s in _ml_predict(text, category=category, job_title=job_title): | |
| _add(s) | |
| return sorted(found.values()) | |
| def extract_soft_skills(text: str) -> list: | |
| """ | |
| Extract soft skills from resume text. | |
| Sources (deduplicated, case-insensitive): | |
| 1. ESCO transversal-skill hits via JobBERT-v2 (data/esco_transversal_uris.txt) | |
| 2. The legacy 20-word _SOFT_SKILLS regex denylist | |
| """ | |
| if not text: | |
| return [] | |
| found: dict[str, str] = {} # lower-case key → display label | |
| def _add(label: str) -> None: | |
| if not label: | |
| return | |
| k = label.lower() | |
| if k not in found: | |
| found[k] = label | |
| # Layer 1: ESCO transversal hits (JobBERT semantic match) | |
| if _ESCO_OK: | |
| _hard, soft_hits = _esco_extract_skills(text) | |
| for s in soft_hits: | |
| _add(s) | |
| # Layer 2: regex denylist (catches obvious surface forms ESCO may miss) | |
| text_lower = text.lower() | |
| for skill in _SOFT_SKILLS: | |
| if re.search(r'\b' + re.escape(skill) + r'\b', text_lower): | |
| _add(skill.title()) | |
| return sorted(found.values()) | |
| # ============================================================================= | |
| # 3. WORK EXPERIENCE | |
| # ============================================================================= | |
| def _parse_date_safe(date_str: str): | |
| """ | |
| Safely parse a date string. Returns datetime or None. | |
| """ | |
| try: | |
| return dateparser.parse(date_str, default=datetime(2000, 1, 1)) | |
| except Exception: | |
| return None | |
| def _merge_date_ranges(ranges: list) -> list: | |
| """ | |
| Given a list of (start_datetime, end_datetime) pairs, sort them by start | |
| date and merge any overlapping or adjacent intervals. | |
| Example: | |
| [(Jan-2020, Jun-2021), (Mar-2021, Dec-2022)] | |
| → [(Jan-2020, Dec-2022)] ← correctly avoids double-counting overlap | |
| Returns the merged list sorted by start date. | |
| """ | |
| if not ranges: | |
| return [] | |
| sorted_r = sorted(ranges, key=lambda x: x[0]) | |
| merged = [list(sorted_r[0])] # work with mutable lists | |
| for start, end in sorted_r[1:]: | |
| prev_start, prev_end = merged[-1] | |
| if start <= prev_end: # overlapping or touching | |
| merged[-1][1] = max(prev_end, end) | |
| else: | |
| merged.append([start, end]) | |
| return [(s, e) for s, e in merged] | |
| # Section-header keyword maps. Used by _build_section_map to label every | |
| # character offset in the resume with the section it belongs to, so date | |
| # ranges can be classified by SECTION (not by ±N-char window, which bleeds | |
| # across section boundaries in dense CVs). | |
| _SECTION_HEADERS = { | |
| "work": [ | |
| "professional experience", "work experience", "employment history", | |
| "career history", "experience", "work history", "professional summary", | |
| "professional history", "employment", "career", | |
| ], | |
| "internship": [ | |
| "internship", "internships", "internship experience", | |
| "summer internship", "summer training", "industrial training", | |
| "training", "apprenticeship", "trainings & internships", | |
| ], | |
| "education": [ | |
| "education", "academic", "qualifications", "academics", | |
| "educational background", "academic background", "academic qualification", | |
| ], | |
| "projects": [ | |
| "projects", "project", "key projects", "academic projects", | |
| "personal projects", "selected projects", | |
| ], | |
| "certifications": [ | |
| "certifications", "certificates", "courses", "credentials", | |
| ], | |
| "achievements": [ | |
| "achievements", "awards", "honors", "honours", "recognitions", | |
| "publications", "patents", | |
| ], | |
| } | |
| # Header line pattern — short, mostly-uppercase or title-case, optionally | |
| # followed by colon or dash. Real-world CV headers are ≤40 chars. | |
| _HEADER_LINE_RE = re.compile( | |
| r'^\s*([A-Z][A-Za-z &/]{2,40})\s*[:\-–—]?\s*$' | |
| ) | |
| def _build_section_map(text: str) -> list: | |
| """ | |
| Walk the resume line-by-line and emit a list of | |
| (char_offset, section_kind) | |
| tuples marking where each section starts. `section_kind` is one of | |
| "work" / "internship" / "education" / "projects" / "certifications" | |
| / "achievements" / "unknown". The list is sorted by offset. | |
| To classify a date range, binary-search this list for the last entry | |
| whose offset is ≤ the range's start — that entry's kind is the section | |
| the range belongs to. | |
| Falls back to "unknown" for content before any header. | |
| """ | |
| sections: list = [(0, "unknown")] | |
| offset = 0 | |
| for raw_line in text.splitlines(keepends=True): | |
| stripped = raw_line.strip() | |
| line_len = len(raw_line) | |
| m = _HEADER_LINE_RE.match(stripped) if stripped else None | |
| if m: | |
| label = m.group(1).strip().lower() | |
| kind = "unknown" | |
| for sect_kind, keywords in _SECTION_HEADERS.items(): | |
| # Match if the header IS one of the keywords (allow short | |
| # extras like "PROFESSIONAL EXPERIENCE & PROJECTS") | |
| if any(label == k or label.startswith(k + " ") | |
| or k in label.split() | |
| for k in keywords): | |
| kind = sect_kind | |
| break | |
| if kind != "unknown": | |
| sections.append((offset, kind)) | |
| offset += line_len | |
| return sections | |
| def _section_at(sections: list, char_offset: int) -> str: | |
| """Return the section kind covering this offset (binary-search-friendly).""" | |
| last = "unknown" | |
| for off, kind in sections: | |
| if off <= char_offset: | |
| last = kind | |
| else: | |
| break | |
| return last | |
| # Conservative fallback regexes — used ONLY when no section header was found | |
| # anywhere in the CV (rare; mostly bullet-only PDFs). They look in a small | |
| # ±80-char window so we don't pull "Trainee" / "Project" from a neighbouring | |
| # section. | |
| _INTERNSHIP_INLINE_RE = re.compile( | |
| r'\b(intern(?:ship)?s?|summer\s+training|industrial\s+training|' | |
| r'apprentice|articled\s+assistant|practicum|co[-\s]?op)\b', | |
| re.IGNORECASE, | |
| ) | |
| _NON_WORK_INLINE_RE = re.compile( | |
| r'\b(bachelor|master|b\.?\s?tech|m\.?\s?tech|b\.?\s?e\.?|m\.?\s?e\.?|' | |
| r'b\.?\s?sc|m\.?\s?sc|mba|pgdm|ph\.?d|diploma|class\s+(?:x|xii|10|12)|' | |
| r'high\s+school|cbse|icse|university|college|institute|school|' | |
| r'certification|certificate|coursera|udemy|nptel|edx|hackerrank|' | |
| r'final[-\s]?year\s+project|capstone|thesis|publication|patent|' | |
| r'hackathon|olympiad)\b', | |
| re.IGNORECASE, | |
| ) | |
| def _classify_date_range(text: str, span_start: int, span_end: int, | |
| sections: list) -> str: | |
| """ | |
| Decide whether a date range belongs to a full-time role, an internship, | |
| or neither (education / project / award). | |
| Strategy: | |
| 1. Look up which SECTION the range falls into using the pre-computed | |
| section map. Sections are derived from real header lines, so this | |
| is dramatically more accurate than a sliding window. | |
| 2. If the section is "unknown" (no header found, or content before | |
| the first header), fall back to a tight inline keyword window. | |
| Returns one of: "work" | "internship" | "non_work" | |
| """ | |
| sect = _section_at(sections, span_start) | |
| if sect == "internship": | |
| return "internship" | |
| if sect in ("education", "projects", "certifications", "achievements"): | |
| return "non_work" | |
| if sect == "work": | |
| return "work" | |
| # Unknown section — use a tight inline window | |
| lo = max(0, span_start - 80) | |
| hi = min(len(text), span_end + 80) | |
| window = text[lo:hi] | |
| if _NON_WORK_INLINE_RE.search(window): | |
| return "non_work" | |
| if _INTERNSHIP_INLINE_RE.search(window): | |
| return "internship" | |
| return "work" | |
| # Master date-range regex — captures ALL of: | |
| # Jan 2020 – Mar 2023 / January 2020 to March 2023 | |
| # Jan'20 – Mar'23 / 01/2020 – 03/2023 | |
| # 01-2020 – 03-2023 / 2020 – 2023 | |
| # May 2021 – Present / Jun 2022 – Current / Aug 2024 – Ongoing | |
| _MONTH_TOKEN = ( | |
| r'(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|' | |
| r'Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|' | |
| r'Nov(?:ember)?|Dec(?:ember)?)' | |
| ) | |
| _DATE_START_TOKEN = ( | |
| rf'(?:{_MONTH_TOKEN}[\s,\.-]*\'?\d{{2,4}}' # Jan 2020 / Jan'20 | |
| rf'|(?:0?[1-9]|1[0-2])[\s/.\-](?:19|20)\d{{2}}' # 01/2020 | |
| rf'|(?:19|20)\d{{2}})' # 2020 | |
| ) | |
| _DATE_END_TOKEN = ( | |
| rf'(?:{_MONTH_TOKEN}[\s,\.-]*\'?\d{{2,4}}' | |
| rf'|(?:0?[1-9]|1[0-2])[\s/.\-](?:19|20)\d{{2}}' | |
| rf'|(?:19|20)\d{{2}}' | |
| rf'|[Pp]resent|[Cc]urrent|[Nn]ow|[Tt]ill\s+[Dd]ate|[Oo]ngoing)' | |
| ) | |
| _DATE_RANGE_RE = re.compile( | |
| rf'({_DATE_START_TOKEN})\s*(?:[-–—]|to|until|till)\s*({_DATE_END_TOKEN})', | |
| re.IGNORECASE, | |
| ) | |
| def _scan_date_ranges(text: str) -> list: | |
| """ | |
| Scan the ENTIRE resume text and return a list of dicts: | |
| {start: datetime, end: datetime, kind: "work"|"internship"|"non_work", span: (lo, hi)} | |
| Each date range is classified by the SECTION header it falls under | |
| (Experience, Internship, Education, Projects, Certifications, …) — not | |
| by a sliding text window — so section boundaries are respected even | |
| when adjacent sections are physically close on the page. | |
| Date ranges shorter than 1 month or longer than 50 years are discarded | |
| as parser noise. Duplicates (same month/year start AND end) are kept | |
| only once so the same line being matched twice doesn't double-count. | |
| """ | |
| now = datetime.now() | |
| sections = _build_section_map(text) | |
| out = [] | |
| seen = set() | |
| for m in _DATE_RANGE_RE.finditer(text): | |
| start_str, end_str = m.group(1).strip(), m.group(2).strip() | |
| start = _parse_date_safe(start_str) | |
| if not start: | |
| continue | |
| if end_str.lower() in ('present', 'current', 'now', 'till date', | |
| 'ongoing'): | |
| end = now | |
| else: | |
| end = _parse_date_safe(end_str) | |
| if not end or end < start: | |
| continue | |
| delta = relativedelta(end, start) | |
| months = delta.years * 12 + delta.months | |
| if not (0 < months < 600): | |
| continue | |
| key = (start.year, start.month, end.year, end.month) | |
| if key in seen: | |
| continue | |
| seen.add(key) | |
| kind = _classify_date_range(text, m.start(), m.end(), sections) | |
| out.append({"start": start, "end": end, "kind": kind, | |
| "span": (m.start(), m.end())}) | |
| return out | |
| def _sum_months(pairs: list) -> int: | |
| """Sum non-overlapping calendar months across (start, end) pairs.""" | |
| merged = _merge_date_ranges(pairs) | |
| total = 0 | |
| for s, e in merged: | |
| d = relativedelta(e, s) | |
| total += d.years * 12 + d.months | |
| return total | |
| def extract_experience(text: str) -> dict: | |
| """ | |
| Extract work-experience details from FULL resume text. | |
| Strategy (no length cap — scans every character of the input): | |
| 1. spaCy ORG entities for company names (kept short-context for speed). | |
| 2. Regex sweep for ALL date ranges in the text, including those buried | |
| several pages deep. | |
| 3. Each date range is classified by surrounding context as | |
| "work" / "internship" / "non_work" (education, project, award, etc.) | |
| using ±180-char keyword windows. This is what lets us separate the | |
| "full-time experience" total from "internship experience" cleanly. | |
| 4. Total work experience = SUM of non-overlapping merged work ranges | |
| (so two overlapping roles don't double-count), with internship and | |
| non-work ranges excluded entirely. | |
| Returns: | |
| { | |
| "total_years" : 4.5, | |
| "internship_years" : 0.5, | |
| "job_titles" : [...], | |
| "companies" : [...], | |
| "avg_tenure_months" : 22.0, | |
| "last_company" : "TCS", | |
| "stability_flag" : False, | |
| } | |
| """ | |
| result = { | |
| "total_years" : 0.0, | |
| "internship_years" : 0.0, | |
| "job_titles" : [], | |
| "companies" : [], | |
| "avg_tenure_months" : 0.0, | |
| "last_company" : "", | |
| "stability_flag" : False, | |
| } | |
| # --- Extract company names via spaCy ORG entities --- | |
| # Companies often appear in the first 1-2 pages, but we scan the first | |
| # 6000 chars (≈2-3 pages) to balance speed and completeness. The LLM | |
| # extractor sees the full text and remains the primary source. | |
| companies = [] | |
| if _SPACY_OK and _NLP: | |
| doc = _NLP(text[:6000]) | |
| for ent in doc.ents: | |
| if ent.label_ == "ORG": | |
| name = ent.text.strip() | |
| if (len(name) >= 2 and | |
| not name.isdigit() and | |
| name not in companies): | |
| companies.append(name) | |
| result["companies"] = companies[:10] | |
| result["last_company"] = companies[0] if companies else "" | |
| # --- Extract job titles via common patterns --- | |
| title_patterns = [ | |
| r'(?i)(senior|junior|lead|principal|associate|staff)?\s*' | |
| r'(software|data|ml|ai|backend|frontend|full.?stack|devops|cloud|' | |
| r'product|project|hr|talent|recruitment|business|marketing|sales|' | |
| r'finance|operations|' | |
| r'analyst|engineer|developer|architect|manager|consultant|' | |
| r'scientist|specialist|coordinator|executive|intern|trainee)', | |
| ] | |
| titles = [] | |
| for pattern in title_patterns: | |
| matches = re.findall(pattern, text) | |
| for match in matches: | |
| title = ' '.join(m for m in match if m).strip().title() | |
| if title and title not in titles and len(title) > 4: | |
| titles.append(title) | |
| result["job_titles"] = titles[:8] | |
| # --- Scan ALL date ranges from the full text and classify each one --- | |
| all_ranges = _scan_date_ranges(text) | |
| work_pairs = [(r["start"], r["end"]) for r in all_ranges | |
| if r["kind"] == "work"] | |
| intern_pairs = [(r["start"], r["end"]) for r in all_ranges | |
| if r["kind"] == "internship"] | |
| if work_pairs: | |
| work_months = _sum_months(work_pairs) | |
| individual_months = [] | |
| for s, e in work_pairs: | |
| d = relativedelta(e, s) | |
| m = d.years * 12 + d.months | |
| if 0 < m < 600: | |
| individual_months.append(m) | |
| avg_months = (sum(individual_months) / len(individual_months) | |
| if individual_months else 0) | |
| result["total_years"] = round(work_months / 12, 1) | |
| result["avg_tenure_months"] = round(avg_months, 1) | |
| result["stability_flag"] = avg_months < config.MIN_TENURE_MONTHS | |
| elif not all_ranges: | |
| # No date ranges anywhere — last-ditch "X years experience" hint | |
| yr_match = re.search( | |
| r'(\d+(?:\.\d+)?)\s*\+?\s*years?\s*(?:of\s*)?(?:experience|exp)', | |
| text, re.IGNORECASE, | |
| ) | |
| if yr_match: | |
| result["total_years"] = float(yr_match.group(1)) | |
| if intern_pairs: | |
| result["internship_years"] = round(_sum_months(intern_pairs) / 12, 1) | |
| return result | |
| # ============================================================================= | |
| # 4. EDUCATION | |
| # ============================================================================= | |
| def extract_education(text: str) -> dict: | |
| """ | |
| Extract highest education qualification from resume text. | |
| Returns: | |
| {"degree": "B.Tech", "field": "Computer Science", "institution": "VIT"} | |
| """ | |
| result = {"degree": "", "field": "", "institution": ""} | |
| text_lower = text.lower() | |
| # Find degree | |
| for deg in sorted(_DEGREE_KEYWORDS, key=len, reverse=True): | |
| if re.search(r'\b' + re.escape(deg) + r'\b', text_lower): | |
| result["degree"] = deg.upper() if len(deg) <= 5 else deg.title() | |
| break | |
| # Find field of study — common patterns after degree keyword | |
| field_match = re.search( | |
| r'(?:b\.?tech|m\.?tech|b\.?sc|m\.?sc|bachelor|master|mba|pgdm)' | |
| r'[,\s]+(?:in\s+)?([a-zA-Z\s&/]{3,40}?)(?:\n|,|\.|from|at|–|-)', | |
| text, re.IGNORECASE | |
| ) | |
| if field_match: | |
| result["field"] = field_match.group(1).strip().title() | |
| # Find institution via spaCy ORG or keyword pattern | |
| inst_match = re.search( | |
| r'\b(?:university|college|institute|iit\s?\w*|nit\s?\w*|bits|iim|vit|' | |
| r'symbiosis|amity|manipal|lpu)\b[a-zA-Z\s,\.]{0,40}', | |
| text, re.IGNORECASE | |
| ) | |
| if inst_match: | |
| result["institution"] = inst_match.group(0).strip().title() | |
| elif _SPACY_OK and _NLP: | |
| # Find ORG entities near degree keywords | |
| edu_section = _find_section(text, ["education", "academic", "qualification"]) | |
| if edu_section: | |
| doc = _NLP(edu_section[:500]) | |
| for ent in doc.ents: | |
| if ent.label_ == "ORG" and len(ent.text) > 3: | |
| result["institution"] = ent.text.strip().title() | |
| break | |
| return result | |
| # ============================================================================= | |
| # 5. PROJECTS, CERTIFICATIONS, HOBBIES | |
| # ============================================================================= | |
| def extract_projects(text: str) -> list: | |
| """ | |
| Extract project names/descriptions from the Projects section. | |
| Returns: | |
| List of project name strings (up to 8). | |
| """ | |
| section = _find_section(text, ["projects", "project work", "key projects"]) | |
| if not section: | |
| return [] | |
| # Each project usually starts with a title on its own line or after a bullet | |
| projects = [] | |
| lines = section.split('\n') | |
| for line in lines: | |
| line = line.strip().lstrip('•–-*►▪').strip() | |
| # Project line: reasonably short, has uppercase, not just a date | |
| if (5 < len(line) < 100 and | |
| not re.match(r'^\d{4}', line) and | |
| any(c.isupper() for c in line)): | |
| projects.append(line) | |
| return projects[:8] | |
| def extract_certifications(text: str) -> list: | |
| """ | |
| Extract certifications from resume text. | |
| Returns: | |
| List of certification name strings (up to 10). | |
| """ | |
| section = _find_section(text, | |
| ["certifications", "certificates", "courses", | |
| "training", "achievements"]) | |
| source = section if section else text | |
| # Common certification keywords | |
| cert_patterns = [ | |
| r'(?:AWS|Azure|GCP|Google|Microsoft|Oracle|Cisco|PMI|PMP|' | |
| r'Scrum|SAFe|ITIL|CompTIA|Coursera|Udemy|edX|NPTEL|' | |
| r'HackerRank|HackerEarth|Certified|Certification)' | |
| r'[a-zA-Z\s\-:®™]{0,60}', | |
| ] | |
| certs = [] | |
| for pattern in cert_patterns: | |
| matches = re.findall(pattern, source) | |
| for m in matches: | |
| m = m.strip().rstrip(':-') | |
| if len(m) > 5 and m not in certs: | |
| certs.append(m) | |
| return certs[:10] | |
| def extract_hobbies(text: str) -> list: | |
| """ | |
| Extract hobbies and interests from resume text. | |
| Returns: | |
| List of hobby strings. | |
| """ | |
| section = _find_section(text, ["hobbies", "interests", "activities", | |
| "extra-curricular", "extracurricular"]) | |
| if not section: | |
| return [] | |
| hobbies = [] | |
| for line in section.split('\n'): | |
| line = line.strip().lstrip('•–-*').strip() | |
| if 2 < len(line) < 60: | |
| # Split by commas too (e.g. "Reading, Music, Cricket") | |
| parts = [p.strip() for p in line.split(',')] | |
| for p in parts: | |
| if p and p not in hobbies: | |
| hobbies.append(p.title()) | |
| return hobbies[:10] | |
| # ============================================================================= | |
| # 6. RECENCY FLAG | |
| # ============================================================================= | |
| def extract_recency(text: str, skills: list) -> str: | |
| """ | |
| Determine how recently the candidate used their key skills. | |
| Logic: | |
| - Find the most recent employment date in the resume | |
| - If last job ended within RECENCY_STALE_YEARS → "Recent" | |
| - If last job ended more than RECENCY_STALE_YEARS ago → "Stale" | |
| - If no dates found → "Unknown" | |
| Args: | |
| text: Raw resume text | |
| skills: Extracted skills list (unused for now, reserved for future) | |
| Returns: | |
| "Recent" | "Stale" | "Unknown" | |
| """ | |
| now = datetime.now() | |
| most_recent = None | |
| # Look for "Present" / "Current" — candidate is currently employed | |
| if re.search(r'\b(?:present|current|now)\b', text, re.IGNORECASE): | |
| return "Recent" | |
| # Find all 4-digit years mentioned in the resume | |
| years = re.findall(r'\b(20\d{2})\b', text) | |
| if years: | |
| latest_year = max(int(y) for y in years) | |
| years_ago = now.year - latest_year | |
| if years_ago <= config.RECENCY_STALE_YEARS: | |
| return "Recent" | |
| else: | |
| return "Stale" | |
| return "Unknown" | |
| # ============================================================================= | |
| # INTERNAL HELPER — Section Finder | |
| # ============================================================================= | |
| def _find_section(text: str, keywords: list) -> str: | |
| """ | |
| Find a named section in a resume and return its content. | |
| Looks for lines that match section header keywords, then returns | |
| everything until the next section header (all-caps or title-case line). | |
| Args: | |
| text: Full resume text | |
| keywords: List of keywords to identify the section | |
| Returns: | |
| Section content string, or empty string if not found. | |
| """ | |
| lines = text.split('\n') | |
| section_lines = [] | |
| in_section = False | |
| # Section header pattern — all caps or title case, short line | |
| header_pattern = re.compile(r'^[A-Z][A-Z\s&/]{2,30}$') | |
| for i, line in enumerate(lines): | |
| stripped = line.strip() | |
| # Detect section start | |
| if any(kw.lower() in stripped.lower() for kw in keywords): | |
| in_section = True | |
| continue | |
| if in_section: | |
| # Stop at next section header | |
| if (header_pattern.match(stripped) and | |
| len(stripped) > 3 and | |
| stripped not in ['AND', 'OR', 'THE']): | |
| break | |
| section_lines.append(line) | |
| return '\n'.join(section_lines).strip() | |
| # ============================================================================= | |
| # LLM EXTRACTION — Primary method for field labelling | |
| # ============================================================================= | |
| # Uses Groq (free) or OpenAI to intelligently label every field from raw text. | |
| # Falls back to regex-based extraction when no API key is available. | |
| # LLM extraction is significantly more reliable because it understands context — | |
| # it can tell "Python" in "fluent in Python and Java" is a skill, and | |
| # "Python" in "born in Python, Mississippi" is not. | |
| _LLM_EXTRACTOR_OK = False | |
| _llm_extractor_client = None | |
| _llm_extractor_model = "" | |
| def user_config_dir() -> str: | |
| """ | |
| OS-standard per-user config directory for keys. | |
| Windows: %LOCALAPPDATA%\\ResumeAnalyzer | |
| macOS : ~/Library/Application Support/ResumeAnalyzer | |
| Linux : $XDG_CONFIG_HOME/resume_analyzer (or ~/.config/resume_analyzer) | |
| """ | |
| if sys.platform == "win32": | |
| base = os.environ.get("LOCALAPPDATA") or os.path.expanduser("~") | |
| return os.path.join(base, "ResumeAnalyzer") | |
| if sys.platform == "darwin": | |
| return os.path.expanduser("~/Library/Application Support/ResumeAnalyzer") | |
| base = os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config") | |
| return os.path.join(base, "resume_analyzer") | |
| def user_env_path() -> str: | |
| return os.path.join(user_config_dir(), ".env") | |
| def _load_user_api_keys() -> None: | |
| """ | |
| Read API keys from the per-user .env file (if it exists) and inject them | |
| into os.environ — but ONLY for variables not already set. Environment | |
| variables always win, so devs running locally keep using their shell env. | |
| """ | |
| path = user_env_path() | |
| if not os.path.exists(path): | |
| return | |
| try: | |
| with open(path, encoding="utf-8") as f: | |
| for line in f: | |
| line = line.strip() | |
| if not line or line.startswith("#") or "=" not in line: | |
| continue | |
| key, _, val = line.partition("=") | |
| key = key.strip() | |
| val = val.strip().strip('"').strip("'") | |
| if key and val and key not in os.environ: | |
| os.environ[key] = val | |
| except Exception as exc: | |
| print(f"[Extractor] couldn't read {path}: {type(exc).__name__}: {exc}") | |
| def _init_llm_extractor(): | |
| """ | |
| Initialise LLM client for extraction. Priority order: | |
| 1. Anthropic Claude (best quality — install: pip install anthropic) | |
| 2. Groq via OpenAI-compatible API (openai package already installed) | |
| 3. OpenAI directly | |
| """ | |
| global _LLM_EXTRACTOR_OK, _llm_extractor_client, _llm_extractor_model | |
| # User-config fallback: when the app is installed via the Windows installer | |
| # there's no project-level .env. The Settings UI writes per-user keys to | |
| # the OS's standard user-config directory; honour those if the environment | |
| # doesn't already define them. | |
| _load_user_api_keys() | |
| # ── Option 1: Anthropic Claude (best at understanding context) ──────────── | |
| anthropic_key = os.environ.get("ANTHROPIC_API_KEY", "") | |
| if anthropic_key: | |
| try: | |
| import anthropic as _anthropic | |
| _llm_extractor_client = _anthropic.Anthropic(api_key=anthropic_key) | |
| _llm_extractor_model = "claude-haiku-4-5-20251001" | |
| _LLM_EXTRACTOR_OK = True | |
| print("[Extractor] LLM: Anthropic Claude (claude-haiku-4-5)") | |
| return | |
| except ImportError: | |
| print("[Extractor] anthropic package not installed — pip install anthropic") | |
| # ── Option 2: Groq via OpenAI-compatible API (no groq package needed) ──── | |
| groq_key = os.environ.get("GROQ_API_KEY", "") | |
| if groq_key: | |
| try: | |
| from openai import OpenAI | |
| _llm_extractor_client = OpenAI( | |
| api_key = groq_key, | |
| base_url = "https://api.groq.com/openai/v1", | |
| ) | |
| _llm_extractor_model = "llama-3.3-70b-versatile" | |
| _LLM_EXTRACTOR_OK = True | |
| print("[Extractor] LLM: Groq llama-3.3-70b-versatile (OpenAI-compat)") | |
| return | |
| except ImportError: | |
| print("[Extractor] openai package not installed") | |
| # ── Option 3: OpenAI directly ───────────────────────────────────────────── | |
| openai_key = os.environ.get("OPENAI_API_KEY", "") | |
| if openai_key: | |
| try: | |
| from openai import OpenAI | |
| _llm_extractor_client = OpenAI(api_key=openai_key) | |
| _llm_extractor_model = "gpt-4o-mini" | |
| _LLM_EXTRACTOR_OK = True | |
| print("[Extractor] LLM: OpenAI gpt-4o-mini") | |
| return | |
| except ImportError: | |
| pass | |
| print("[Extractor] WARNING: No LLM API key found — falling back to regex extraction only.") | |
| print(" Set GROQ_API_KEY in .env for free LLM-powered extraction.") | |
| _init_llm_extractor() | |
| _LLM_EXTRACTION_PROMPT = """Extract structured information from this resume text. | |
| Return ONLY valid JSON. No explanation, no markdown fences. | |
| Extract these fields: | |
| - candidate_name: The person's full name ONLY (given name + surname). Must be a real person's name found in the first 3 lines. NOT a job title (e.g. NOT "Financial Controller.txt"), NOT a location (e.g. NOT "Delhi NCR"), NOT a company name. (string) | |
| - email: Email address (string or "") | |
| - phone: Phone number digits only (string or "") | |
| - age: Age in years if mentioned, else 0 (number) | |
| - location: City and state/country if mentioned (string or "") | |
| - linkedin_url: LinkedIn profile URL if present (string or "") | |
| - github_url: GitHub profile URL if present, e.g. github.com/username (string or "") | |
| - portfolio_url: Personal portfolio / personal website URL if present (must NOT be linkedin.com or github.com — those go in their own fields) (string or "") | |
| - resume_summary: Write a 2-3 sentence professional summary of this candidate based on their overall profile. Include their years of experience, domain, strongest skills, and any standout achievement. (string) | |
| - technical_skills: List of technical/hard skills only (list of strings) | |
| - soft_skills: List of interpersonal/behavioural skills (list of strings) | |
| - languages: List of spoken/written languages the candidate knows (list of strings) | |
| - total_experience_years: Total years of full-time work experience, NOT counting internships (number) | |
| - job_titles: All full-time job titles held, most recent first (list of strings) | |
| - companies: List of REAL employer organisation names ONLY — i.e. the legally-registered entity the candidate was paid by. Each entry MUST look like a company brand. | |
| ✓ GOOD: "Infosys", "HDFC Bank", "Groz Engineering Tools Pvt. Ltd.", "Tata Consultancy Services", "Indian Armour Systems Pvt. Ltd." | |
| ✗ BAD — NEVER include any of these as a "company": | |
| - Cities/regions: "Delhi-NCR", "Mumbai", "Bangalore", "Gurgaon" | |
| - Job functions/departments: "Operations", "Insurance Operations", "Credit Management", "Business Analysis", "Operations Management", "Treasury & Cash Management" | |
| - Section headers from the resume: "EXPERIENCE", "PROFESSIONAL EXPERIENCE", "Work History" | |
| - Bullet-point fragments: "• Executed daily reviews", "Handled client queries", "Led a team of 5" | |
| - Acronyms-only: "KPI", "API", "PROGRAM &" | |
| (list of strings) | |
| - last_company: The MOST RECENT employer's organisation name ONLY. MUST be a real registered company (e.g. "HDFC Bank", "Infosys", "Groz Engineering Tools Pvt. Ltd."). MUST NOT be a city, department, job function, or section header. If you cannot identify a clear company name, return "". (string or "") | |
| - company_tenures: For every full-time role (NOT internships, NOT projects), one entry formatted EXACTLY as: "Company Name | Role Title | Start – End". Use full month names + 4-digit years when the resume gives them (e.g. "May 2021 – March 2023"); use year-only when months are not stated (e.g. "2018 – 2021"). Use the literal word "Present" for current/ongoing roles — do NOT compute or substitute the current year, the application will handle that. Example outputs: "Groz Engineering Tools Pvt. Ltd. | Production Engineer | Oct 2014 – Feb 2017", "HDFC Bank | Branch Manager | 2018 – 2021", "TCS | Senior Data Analyst | Jun 2022 – Present". Most recent role first. Company name MUST NOT include the city. (list of strings) | |
| - avg_tenure_months: Average months per full-time role (number or 0) | |
| - internships: List of internship descriptions — each as "Company | Role | Duration" (list of strings) | |
| - internship_count: Number of internships (number) | |
| - education_degree: Highest degree e.g. B.Tech, MBA, M.Tech (string or "") | |
| - education_field: Field of study e.g. Computer Science (string or "") | |
| - education_institution: University or college name (string or "") | |
| - graduation_year: Year of most recent graduation (number or 0) | |
| - certifications: List of certification names (list of strings) | |
| - projects: TECHNICAL PROJECTS the candidate personally built/designed/researched. A project is a tangible deliverable like an app, model, dashboard, system, or research study. STRICT RULES: (a) NEVER list employers/companies here — those belong in `companies`. (b) NEVER list job descriptions or work-history bullets here. (c) Each entry must look like a project title with an optional description, e.g. "Sales Revenue Dashboard — Automated monthly reporting using Python" or "Customer Churn Prediction Model — Reduced churn by 12%". If the resume has no clearly-labelled Projects section, return an empty list []. (list of strings) | |
| - achievements: List of awards, recognitions, hackathon wins, publications (list of strings) | |
| - hobbies: List of hobbies and interests (list of strings) | |
| - recency_flag: "Recent" if currently employed or worked within 2 years, "Stale" if last job was more than 2 years ago, "Unknown" if unclear | |
| - pros: Top 3 strengths of this candidate as a recruiter would see them (list of 3 strings) | |
| - cons: Top 3 weaknesses or gaps in this resume (list of 3 strings) | |
| Resume text: | |
| {resume_text} | |
| Return ONLY the JSON object. Every field must be present — use "" or [] or 0 for missing values.""" | |
| def _extract_via_llm(raw_text: str) -> dict: | |
| """ | |
| Use LLM to extract all structured fields from resume text. | |
| Returns a flat dict with all fields, or empty dict on failure. | |
| The caller merges this with regex-based fallbacks for any missing fields. | |
| """ | |
| if not _LLM_EXTRACTOR_OK or not _llm_extractor_client: | |
| return {} | |
| # Send the FULL resume text — every modern LLM we use (Claude haiku-4.5 | |
| # ≥200k, Groq llama-3.3-70b ≥128k, gpt-4o-mini ≥128k) has plenty of headroom | |
| # for even a 10-page CV. The previous 6 000-char truncation was the main | |
| # reason late-page employers (page 3+) were silently dropped from | |
| # company_tenures, which broke total-experience accuracy. | |
| # Cap at 60 000 chars (~15 k tokens) as a safety belt for pathological CVs. | |
| text_snippet = raw_text[:60000].strip() | |
| # Retry transient errors (rate limits, brief network blips). For | |
| # per-day token-cap rate limits, falling back to a smaller Groq model | |
| # (8B instant) keeps the pipeline producing usable extractions instead | |
| # of silently giving up on the LLM entirely. | |
| last_exc = None | |
| raw = "" | |
| fallback_models = ( | |
| [_llm_extractor_model, "llama-3.1-8b-instant"] | |
| if "llama-3.3-70b" in _llm_extractor_model | |
| else [_llm_extractor_model] | |
| ) | |
| for model_name in fallback_models: | |
| # The 8B fallback has only ~8k token context — truncate the resume | |
| # text much harder when we're using it, otherwise we get 413s. | |
| if "8b" in model_name.lower(): | |
| snippet_for_model = text_snippet[:12000] | |
| else: | |
| snippet_for_model = text_snippet | |
| prompt = _LLM_EXTRACTION_PROMPT.format(resume_text=snippet_for_model) | |
| for attempt in range(3): | |
| try: | |
| # Anthropic client has a different API surface | |
| if hasattr(_llm_extractor_client, "messages"): | |
| response = _llm_extractor_client.messages.create( | |
| model = model_name, | |
| max_tokens = 3000, | |
| messages = [{"role": "user", "content": prompt}], | |
| ) | |
| raw = response.content[0].text.strip() | |
| else: | |
| response = _llm_extractor_client.chat.completions.create( | |
| model = model_name, | |
| messages = [{"role": "user", "content": prompt}], | |
| max_tokens = 3000, | |
| temperature = 0.1, | |
| ) | |
| raw = response.choices[0].message.content.strip() | |
| break # success | |
| except Exception as exc: | |
| msg = str(exc) | |
| last_exc = exc | |
| # On daily-quota rate-limits, don't waste retries on the same | |
| # model — break out to the fallback model immediately. | |
| if "tokens per day" in msg or "TPD" in msg: | |
| break | |
| # Per-minute rate-limit: short backoff and retry | |
| wait = 2 ** attempt | |
| m = re.search(r"try again in\s+([\d.]+)s", msg, re.I) | |
| if m: | |
| wait = min(30, float(m.group(1)) + 1.0) | |
| time.sleep(wait) | |
| if raw: | |
| break # found a working model | |
| if not raw: | |
| raise (last_exc or RuntimeError("LLM call failed")) | |
| try: | |
| # Strip accidental markdown fences | |
| raw = re.sub(r"^```(?:json)?|```$", "", raw, flags=re.MULTILINE).strip() | |
| parsed = json.loads(raw) | |
| if not isinstance(parsed, dict): | |
| return {} | |
| # Normalise list fields | |
| list_fields = [ | |
| "technical_skills", "soft_skills", "languages", | |
| "job_titles", "companies", "certifications", | |
| "projects", "hobbies", "pros", "cons", | |
| "internships", "achievements", "company_tenures", | |
| ] | |
| for field in list_fields: | |
| val = parsed.get(field, []) | |
| if not isinstance(val, list): | |
| parsed[field] = [str(val)] if val else [] | |
| return parsed | |
| except json.JSONDecodeError as e: | |
| print(f"[Extractor] LLM returned invalid JSON: {e}") | |
| return {} | |
| except Exception as e: | |
| print(f"[Extractor] LLM extraction failed: {type(e).__name__}: {e}") | |
| return {} | |
| # ============================================================================= | |
| # MASTER FUNCTION — Call This From main.py | |
| # ============================================================================= | |
| def build_resume_record(file_name: str, raw_text: str) -> dict: | |
| """ | |
| Build a complete, DB-ready resume record from raw text. | |
| EXTRACTION STRATEGY — two-pass, best-of-both: | |
| Pass 1 — LLM extraction (Groq or OpenAI): | |
| Sends raw text to an LLM with a structured prompt. | |
| Returns a labelled JSON dict for all fields. | |
| Reliable because LLMs understand context — they can distinguish | |
| "Python (skill)" from "Python (place name)" and correctly identify | |
| which lines are education vs experience vs projects. | |
| Pass 2 — Regex/rule-based fallback: | |
| Runs for any field that LLM left empty or returned invalid data for. | |
| Uses the existing regex extractors as safety net. | |
| Always runs for: skills taxonomy matching (expands LLM skill list), | |
| contact info (email/phone regex is very reliable). | |
| The final record merges both passes — LLM fields take priority, | |
| regex fills gaps. | |
| Args: | |
| file_name : Original filename e.g. "priya_sharma.pdf" | |
| raw_text : Cleaned text from parser.extract_text() | |
| Returns: | |
| Dict matching resumes table schema, ready for DB insert. | |
| All list fields are JSON strings. | |
| """ | |
| # ── Pass 1: LLM extraction ──────────────────────────────────────────────── | |
| llm = _extract_via_llm(raw_text) | |
| # ── Pass 2: Regex fallbacks ─────────────────────────────────────────────── | |
| name_regex = extract_candidate_name(raw_text, file_name) | |
| contact = extract_contact(raw_text) | |
| url_bundle = extract_urls(raw_text) | |
| skills_regex = extract_skills(raw_text) # taxonomy-matched skills | |
| soft_regex = extract_soft_skills(raw_text) | |
| exp_regex = extract_experience(raw_text) | |
| edu_regex = extract_education(raw_text) | |
| projects_regex = extract_projects(raw_text) | |
| certs_regex = extract_certifications(raw_text) | |
| hobbies_regex = extract_hobbies(raw_text) | |
| recency_regex = extract_recency(raw_text, skills_regex) | |
| # ── Merge: LLM takes priority, regex fills gaps ─────────────────────────── | |
| # Identity — name needs careful handling: | |
| # If the filename gives a clean ≥2-word name (e.g. "Gaurav Joshi" from | |
| # "Gaurav_Joshi_Resume pdf.pdf"), that is more reliable than an LLM | |
| # extraction that may read garbled OCR text (e.g. "AURAV OSHI"). | |
| # Rule: prefer filename when it gives ≥2 words AND LLM gives ≤ same words. | |
| # Only let LLM override when it gives strictly MORE words (i.e. a more complete name). | |
| _llm_name = (llm.get("candidate_name") or "").strip() | |
| _filename_name = _name_from_filename(file_name) | |
| _llm_words = len(_llm_name.split()) if _llm_name != "" else 0 | |
| _file_words = len(_filename_name.split()) if _filename_name != "Unknown" else 0 | |
| if _file_words >= 2 and _llm_words <= _file_words: | |
| # Filename gives a solid 2+ word name and LLM gives same or fewer — | |
| # trust the filename (immune to OCR errors in PDF text) | |
| name = _filename_name | |
| elif _llm_name: | |
| # LLM gives more words (more complete) or filename only has 1 word | |
| name = _llm_name | |
| else: | |
| name = name_regex # spaCy / line-scan fallback | |
| age = int(llm.get("age") or 0) | |
| location = (llm.get("location") or "").strip() | |
| # If LLM didn't find a location, fall back to Indian-city database match | |
| if not location: | |
| location = _extract_location_from_text(raw_text) | |
| # URLs — LLM gives only linkedin_url; the regex bundle adds github, | |
| # portfolio and any other clickable URLs found anywhere in the CV. | |
| linkedin = (llm.get("linkedin_url") or "").strip() or url_bundle["linkedin_url"] | |
| github_url = (llm.get("github_url") or "").strip() or url_bundle["github_url"] | |
| portfolio_url = (llm.get("portfolio_url")or "").strip() or url_bundle["portfolio_url"] | |
| other_urls = url_bundle["other_urls"] | |
| # Summary — LLM only (needs full context understanding) | |
| resume_summary = (llm.get("resume_summary") or "").strip() | |
| # Skills — three-layer routing: | |
| # 1. LLM overrides (data/skill_softness_overrides.csv) are authoritative | |
| # for any label they cover. These are recruiter-context labels | |
| # assigned by llama-3.3-70b and (optionally) human-reviewed. | |
| # 2. Rule-based fallback (regex denylist + ESCO transversal lookup) for | |
| # labels not in the overrides. | |
| # 3. Subtractive merge below guarantees no skill appears in both lists. | |
| # The legacy JobBERT softness classifier is loaded but only consulted | |
| # when config.USE_SOFTNESS_CLASSIFIER = True (off by default; the LLM | |
| # overrides are usually better). | |
| llm_skills = [s.strip() for s in llm.get("technical_skills", []) if s.strip()] | |
| llm_soft = [s.strip() for s in llm.get("soft_skills", []) if s.strip()] | |
| rule_soft = list({s.lower(): s for s in soft_regex + llm_soft}.values()) | |
| rule_hard = list({s.lower(): s for s in skills_regex + llm_skills}.values()) | |
| # Apply LLM overrides: pull anything marked soft out of rule_hard, push | |
| # anything marked hard out of rule_soft. "language" and "other" labels | |
| # drop out of the skill lists entirely (they belong in `languages` / | |
| # nowhere). | |
| soft_final: list[str] = [] | |
| hard_final: list[str] = [] | |
| for s in rule_hard: | |
| ov = _SKILL_OVERRIDES.get(s.lower()) | |
| if ov: | |
| if ov["category"] == "soft": soft_final.append(s) | |
| elif ov["category"] == "hard": hard_final.append(s) | |
| # "language" / "other" → discard | |
| else: | |
| hard_final.append(s) | |
| for s in rule_soft: | |
| ov = _SKILL_OVERRIDES.get(s.lower()) | |
| if ov: | |
| if ov["category"] == "soft": soft_final.append(s) | |
| elif ov["category"] == "hard": hard_final.append(s) | |
| else: | |
| soft_final.append(s) | |
| # Optional in-flight classifier promotion for OOV labels. | |
| soft_keys = {s.lower() for s in soft_final} | |
| if _SOFTNESS_CLF is not None and getattr(config, "USE_SOFTNESS_CLASSIFIER", False): | |
| uncovered = [s for s in hard_final | |
| if s.lower() not in _SKILL_OVERRIDES | |
| and s.lower() not in soft_keys] | |
| if uncovered: | |
| for label, p in _classify_softness(uncovered).items(): | |
| if p >= _SOFTNESS_THRESHOLD: | |
| soft_final.append(label) | |
| soft_keys.add(label.lower()) | |
| merged_soft = list({s.lower(): s for s in soft_final}.values()) | |
| merged_skills = list({s.lower(): s for s in hard_final | |
| if s.lower() not in soft_keys}.values()) | |
| # Languages | |
| languages = [s.strip() for s in llm.get("languages", []) if s.strip()] | |
| # Full-time experience. | |
| # The regex scanner runs across the FULL resume text and now classifies | |
| # every date range as "work" / "internship" / "non_work" (education, | |
| # project, award …) using the surrounding context. That gives us a clean | |
| # split between full-time experience and internship time, computed | |
| # against datetime.now() (pipeline-run date) rather than the stale | |
| # "X years experience" string the candidate typed at CV-write time. | |
| regex_total_years = float(exp_regex["total_years"] or 0) | |
| regex_intern_yrs = float(exp_regex["internship_years"] or 0) | |
| total_years = (regex_total_years if regex_total_years > 0 | |
| else float(llm.get("total_experience_years") or 0)) | |
| internship_years = regex_intern_yrs # always trust regex split; LLM has no | |
| # comparable computed field | |
| job_titles = llm.get("job_titles", []) or exp_regex["job_titles"] | |
| companies = llm.get("companies", []) or exp_regex["companies"] | |
| # Scrub LLM hallucinations: remove location-only entries and trailing cities | |
| companies = [c for c in (_clean_company_name(c) for c in companies) if c] | |
| # last_company: try LLM first, then spaCy regex, then first cleaned company | |
| _lc_raw = (llm.get("last_company") or "").strip() | |
| if not _lc_raw: | |
| _lc_raw = exp_regex.get("last_company", "") | |
| last_company = _clean_company_name(_lc_raw) | |
| # If cleaning rejected it (location/function/garbage), fall back to the | |
| # first valid entry in the cleaned companies list. | |
| if not last_company: | |
| # Walk the companies list in order (most-recent-first from LLM) and | |
| # use the first one that is different from the rejected last_company | |
| for co in companies: | |
| if co and co != _lc_raw: | |
| last_company = co | |
| break | |
| avg_tenure = float(llm.get("avg_tenure_months") or | |
| exp_regex["avg_tenure_months"] or 0) | |
| # Internships (separate from full-time experience) | |
| internships = [s.strip() for s in llm.get("internships", []) if s.strip()] | |
| internship_count = int(llm.get("internship_count") or len(internships)) | |
| # Per-company tenure list — "Company | Role | Start – End" | |
| company_tenures = [s.strip() for s in llm.get("company_tenures", []) if s.strip()] | |
| # Regex fallback: LLM often skips this field (token budget / truncation); | |
| # build tenure entries from the raw text when the LLM returns nothing. | |
| if not company_tenures and companies: | |
| company_tenures = _build_tenures_from_text(raw_text, companies) | |
| # Anchor experience years to the pipeline-run date (NOT the CV write date). | |
| # We have two independent sources of truth: | |
| # (a) regex_total_years — full-CV date-range scan with internship-context | |
| # exclusion (set above). | |
| # (b) tenure_years — derived from the LLM's structured | |
| # company_tenures list, which can pick up roles the regex missed | |
| # because the start/end dates were on a different line from the | |
| # company name. | |
| # Take the MAX of the two so we never under-count when one source has a | |
| # blind spot. The exception is when regex strictly excludes a range as | |
| # internship — in that case we still want to trust the regex split. | |
| tenure_years = _years_from_tenures(company_tenures) | |
| if tenure_years > total_years: | |
| total_years = tenure_years | |
| # Education | |
| edu_degree = (llm.get("education_degree") or "").strip() or edu_regex.get("degree", "") | |
| edu_field = (llm.get("education_field") or "").strip() or edu_regex.get("field", "") | |
| edu_inst = (llm.get("education_institution") or "").strip() or edu_regex.get("institution", "") | |
| graduation_year = int(llm.get("graduation_year") or 0) | |
| education = { | |
| "degree" : edu_degree, | |
| "field" : edu_field, | |
| "institution" : edu_inst, | |
| "graduation_year": graduation_year, | |
| } | |
| # Projects, certifications, achievements, hobbies. | |
| # Trust the LLM's empty list when it ran — the prompt is explicit that an | |
| # absent "Projects" section in the CV must return []. Falling back to the | |
| # heuristic regex would invent project entries from random work bullets, | |
| # which is exactly what we want to avoid. | |
| if _LLM_EXTRACTOR_OK and "projects" in llm: | |
| projects = llm.get("projects") or [] | |
| else: | |
| projects = projects_regex | |
| # Scrub: drop any "project" that's actually a company-name/work-history leak | |
| projects = _filter_project_entries(projects, companies) | |
| certs = llm.get("certifications",[]) or certs_regex | |
| achievements = [s.strip() for s in llm.get("achievements", []) if s.strip()] | |
| hobbies = llm.get("hobbies", []) or hobbies_regex | |
| # Recency | |
| recency = (llm.get("recency_flag") or "").strip().title() | |
| if recency not in ("Recent", "Stale", "Unknown"): | |
| recency = recency_regex | |
| pros = [s.strip() for s in llm.get("pros", []) if s.strip()] | |
| cons = [s.strip() for s in llm.get("cons", []) if s.strip()] | |
| # Stability flag | |
| stability_flag = (avg_tenure < config.MIN_TENURE_MONTHS | |
| if avg_tenure > 0 else False) | |
| # ─── Rule-based fallback when LLM is unavailable (rate-limit / error) ────── | |
| # Each list is handled independently so that a strong candidate (no cons) | |
| # and a weak candidate (no pros) are both covered correctly. | |
| skill_count = len(merged_skills) | |
| if not pros: | |
| _pros = [] | |
| # Experience depth | |
| if total_years >= 5: | |
| _pros.append(f"{total_years:.0f}+ years of professional experience") | |
| elif total_years >= 2: | |
| _pros.append(f"{total_years:.0f} years of relevant experience") | |
| elif total_years >= 1: | |
| _pros.append(f"{total_years:.0f} year of work experience") | |
| # Skills breadth | |
| if skill_count >= 10: | |
| _pros.append(f"Strong technical breadth — {skill_count} skills detected") | |
| elif skill_count >= 5: | |
| _pros.append(f"{skill_count} technical skills identified") | |
| elif skill_count >= 1: | |
| _pros.append(f"{skill_count} technical skill(s) listed") | |
| # Recency | |
| if recency == "Recent": | |
| _pros.append("Currently employed / recently active") | |
| # Stable career | |
| if avg_tenure >= 18: | |
| _pros.append(f"Stable job history (avg {avg_tenure:.0f} months per role)") | |
| # Certifications | |
| if len(certs) >= 2: | |
| _pros.append(f"{len(certs)} certifications on record") | |
| elif len(certs) == 1: | |
| _pros.append("Holds a professional certification") | |
| # Education | |
| if edu_degree: | |
| _pros.append("Holds a " + edu_degree + (f" in {edu_field}" if edu_field else "")) | |
| # Catch-all — always ensure at least one entry | |
| if not _pros: | |
| _pros.append("Resume received — limited extractable data; manual review recommended") | |
| pros = _pros | |
| if not cons: | |
| _cons = [] | |
| # Experience gap | |
| if total_years == 0: | |
| _cons.append("No quantifiable work experience detected in resume") | |
| elif total_years < 2: | |
| _cons.append(f"Limited work experience ({total_years:.1f} yrs) — under typical threshold") | |
| # Skills gap | |
| if skill_count < 3: | |
| _cons.append("Very few technical skills detected — resume may need more detail") | |
| # Job hopping | |
| if stability_flag: | |
| _cons.append(f"Frequent job changes detected (avg {avg_tenure:.0f} months per role)") | |
| # Stale / inactive | |
| if recency == "Stale": | |
| _cons.append("Employment gap or outdated recent role detected") | |
| # No certifications for junior profiles | |
| if not certs and total_years < 3: | |
| _cons.append("No professional certifications listed") | |
| # Catch-all — strong candidates genuinely have no concerns | |
| if not _cons: | |
| _cons.append("No significant concerns identified from extracted resume data") | |
| cons = _cons | |
| # ─── Rule-based summary fallback ───────────────────────────────────────── | |
| if not resume_summary: | |
| _parts = [] | |
| if name and name != "Unknown": | |
| _parts.append(f"{name} is a") | |
| else: | |
| _parts.append("Candidate is a") | |
| if total_years >= 1: | |
| _parts.append(f"{total_years:.0f}-year professional") | |
| else: | |
| _parts.append("professional") | |
| if edu_degree: | |
| _parts.append(f"holding a {edu_degree}" + (f" in {edu_field}" if edu_field else "")) | |
| if merged_skills: | |
| top_skills = merged_skills[:4] | |
| _parts.append(f"with skills in {', '.join(top_skills)}") | |
| if last_company: | |
| _parts.append(f"(most recently at {last_company})") | |
| resume_summary = " ".join(_parts).strip().rstrip(",") + "." | |
| # Capitalise first letter | |
| if resume_summary: | |
| resume_summary = resume_summary[0].upper() + resume_summary[1:] | |
| return { | |
| # Identity | |
| "file_name" : file_name, | |
| "candidate_name" : name, | |
| "email" : contact["email"], | |
| "phone" : contact["phone"], | |
| "age" : age, | |
| "location" : location, | |
| "linkedin_url" : linkedin, | |
| "github_url" : github_url, | |
| "portfolio_url" : portfolio_url, | |
| "other_urls" : json.dumps(other_urls), | |
| # Summary | |
| "resume_summary" : resume_summary, | |
| # Raw text (kept for semantic matching) | |
| "resume_text" : raw_text, | |
| # Skills | |
| "skills" : json.dumps(merged_skills), | |
| "soft_skills" : json.dumps(merged_soft), | |
| "languages" : json.dumps(languages), | |
| # Full-time experience | |
| "experience_years" : total_years, | |
| "last_company" : last_company, | |
| "job_titles" : json.dumps(job_titles), | |
| "companies" : json.dumps(companies), | |
| "avg_tenure_months" : avg_tenure, | |
| # Internships — kept separately so they never inflate the work-exp total | |
| "internships" : json.dumps(internships), | |
| "internship_count" : internship_count, | |
| "internship_years" : internship_years, | |
| # Per-company tenure: ["Company | Role | 18 months", …] | |
| "company_tenures" : json.dumps(company_tenures), | |
| # Education | |
| "education" : json.dumps(education), | |
| "graduation_year" : graduation_year, | |
| # Other | |
| "certifications" : json.dumps(certs), | |
| "projects" : json.dumps(projects), | |
| "achievements" : json.dumps(achievements), | |
| "hobbies" : json.dumps(hobbies), | |
| # Scoring inputs (used by matcher.py) | |
| "recency_flag" : recency, | |
| "stability_flag" : stability_flag, | |
| # Batch | |
| "upload_batch" : datetime.now().strftime("%Y%m%d_%H%M%S"), | |
| "technical_skills": ", ".join(merged_skills), # was "skills" (JSON) | |
| "total_experience_years": total_years, # was "experience_years" | |
| "education_degree": edu_degree, | |
| "education_field": edu_field, | |
| "education_institution": edu_inst, | |
| "pros" : json.dumps(pros), | |
| "cons" : json.dumps(cons), | |
| } | |
| # ============================================================================= | |
| # SELF-TEST | |
| # Run directly: python engine/extractor.py | |
| # ============================================================================= | |
| if __name__ == "__main__": | |
| GREEN = "\033[92m" | |
| RED = "\033[91m" | |
| YELLOW = "\033[93m" | |
| BOLD = "\033[1m" | |
| RESET = "\033[0m" | |
| print(f"\n{BOLD}{'='*60}{RESET}") | |
| print(f"{BOLD} EXTRACTOR SELF-TEST{RESET}") | |
| print(f"{BOLD}{'='*60}{RESET}\n") | |
| # Check spaCy | |
| if _SPACY_OK: | |
| print(f" {GREEN}✓{RESET} spaCy loaded: en_core_web_sm") | |
| else: | |
| print(f" {YELLOW}⚠{RESET} spaCy not available — " | |
| f"run: python -m spacy download en_core_web_sm") | |
| print(f" {GREEN}✓{RESET} Skills taxonomy: {len(_SKILLS_TAXONOMY)} skills loaded\n") | |
| # Sample resume text | |
| SAMPLE = """ | |
| Priya Sharma | |
| priya.sharma@email.com | +91 9876543210 | |
| Mumbai, India | |
| EXPERIENCE | |
| Senior Data Analyst — Infosys, Mumbai | |
| Jan 2022 – Present | |
| - Built sales dashboards using Python, SQL, and Tableau | |
| - Automated reporting pipeline with Pandas and NumPy | |
| Data Analyst — Wipro, Pune | |
| June 2020 – Dec 2021 | |
| - Developed churn prediction model using scikit-learn | |
| - Created Power BI dashboards for leadership team | |
| EDUCATION | |
| B.Tech in Computer Science — VIT University, 2020 | |
| SKILLS | |
| Python, SQL, Tableau, Power BI, Machine Learning, Pandas, NumPy, | |
| scikit-learn, Git, Agile | |
| PROJECTS | |
| Sales Revenue Dashboard — Automated monthly reporting using Python | |
| Customer Churn Model — Reduced churn by 12% using ML classification | |
| CERTIFICATIONS | |
| AWS Cloud Practitioner, Google Data Analytics Certificate | |
| HOBBIES | |
| Reading, Hackathons, Cricket | |
| """ | |
| record = build_resume_record("priya_sharma.pdf", SAMPLE) | |
| passed = 0 | |
| failed = 0 | |
| def check(condition, label, got=""): | |
| global passed, failed | |
| if condition: | |
| print(f" {GREEN}✓{RESET} {label}" | |
| + (f" → {got}" if got else "")) | |
| passed += 1 | |
| else: | |
| print(f" {RED}✗{RESET} {label}" | |
| + (f" → got: {got}" if got else "")) | |
| failed += 1 | |
| check(record["candidate_name"] != "Unknown", | |
| "Name extracted", | |
| record["candidate_name"]) | |
| check("sharma" in record["email"].lower() or "@" in record["email"], | |
| "Email extracted", | |
| record["email"]) | |
| check(len(record["phone"]) >= 8, | |
| "Phone extracted", | |
| record["phone"]) | |
| skills = json.loads(record["skills"]) | |
| check(len(skills) >= 3, | |
| f"Skills extracted ({len(skills)} found)", | |
| ", ".join(skills[:5])) | |
| soft = json.loads(record["soft_skills"]) | |
| check(isinstance(soft, list), | |
| f"Soft skills extracted ({len(soft)} found)", | |
| ", ".join(soft[:3]) if soft else "none") | |
| check(record["experience_years"] > 0, | |
| f"Experience years", | |
| f"{record['experience_years']} yrs") | |
| check(record["avg_tenure_months"] > 0, | |
| f"Avg tenure", | |
| f"{record['avg_tenure_months']} months") | |
| check(record["recency_flag"] in ("Recent", "Stale", "Unknown"), | |
| f"Recency flag", | |
| record["recency_flag"]) | |
| edu = json.loads(record["education"]) | |
| check(edu.get("degree", "") != "", | |
| "Education extracted", | |
| f"{edu.get('degree')} in {edu.get('field')}") | |
| projects = json.loads(record["projects"]) | |
| check(len(projects) >= 1, | |
| f"Projects extracted ({len(projects)} found)", | |
| projects[0] if projects else "none") | |
| certs = json.loads(record["certifications"]) | |
| check(len(certs) >= 1, | |
| f"Certifications extracted ({len(certs)} found)", | |
| certs[0] if certs else "none") | |
| # All required keys present | |
| required_keys = [ | |
| "file_name", "candidate_name", "email", "phone", "resume_text", | |
| "skills", "soft_skills", "experience_years", "last_company", | |
| "job_titles", "education", "certifications", "projects", "hobbies", | |
| "avg_tenure_months", "recency_flag", "upload_batch" | |
| ] | |
| missing_keys = [k for k in required_keys if k not in record] | |
| check(len(missing_keys) == 0, | |
| "All DB schema keys present", | |
| f"Missing: {missing_keys}" if missing_keys else "") | |
| print(f"\n Passed: {GREEN}{passed}{RESET} | " | |
| f"Failed: {RED}{failed}{RESET}") | |
| if failed == 0: | |
| print(f"\n {GREEN}{BOLD}✓ All tests passed — extractor.py is ready.{RESET}") | |
| print(f"\n Next step: tell Claude 'ready for Step 5 (jd_parser)'\n") | |
| else: | |
| print(f"\n {YELLOW}Some tests failed — check output above.{RESET}") | |
| print(f" Partial failures are okay if spaCy isn't installed yet.\n") | |
| print("="*60 + "\n") |