# ============================================================================= # engine/extractor.py — Structured Field Extraction from Resume Text # ============================================================================= # PURPOSE: # Takes raw resume text (output of parser.py) and extracts structured fields. # This is the most important file in the engine — everything downstream # (scoring, explainability) depends on what this file extracts. # # WHAT IT EXTRACTS: # - Candidate name, email, phone # - Technical skills (matched against taxonomy) # - Soft skills # - Work experience (years, roles, companies, tenure, stability) # - Education (degree, field, institution) # - Projects, certifications, hobbies # - Recency flag (how recently key skills were used) # # MAIN FUNCTION (only one you call from outside): # build_resume_record(file_name, raw_text) → dict # This calls all other functions and returns one complete DB-ready dict. # # USAGE: # from engine.extractor import build_resume_record # record = build_resume_record("resume_1.pdf", raw_text) # ============================================================================= import os import re import sys import json import time from datetime import datetime from dateutil import parser as dateparser from dateutil.relativedelta import relativedelta # Add project root to path sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import config # ML skill model — trained on job postings dataset # Falls back silently if model hasn't been trained yet try: from train_skill_model import predict_skills_for_text as _ml_predict _ML_MODEL_OK = True except ImportError: _ML_MODEL_OK = False def _ml_predict(*args, **kwargs): return [] # spaCy — load once at module level so it isn't reloaded for every resume try: import spacy _NLP = spacy.load("en_core_web_sm") _SPACY_OK = True except (ImportError, OSError): _NLP = None _SPACY_OK = False print("[Extractor] WARNING: spaCy model not found. " "Run: python -m spacy download en_core_web_sm") # ============================================================================= # ESCO + JobBERT-v2 SKILL INDEX # ============================================================================= # Built by: python build_esco_index.py # Three artefacts in models/: # esco_embeddings.npy — (N, dim) L2-normalised float32 # esco_skills.json — {labels, altLabels, skillType, model, …} # esco_index_report.json # # If any artefact is missing, the extractor silently falls back to the legacy # taxonomy match + TF-IDF model — so the pipeline never crashes mid-deployment. _ESCO_OK = False _ESCO_MODEL = None _ESCO_EMBEDS = None # numpy (N, dim) _ESCO_META = None # parsed esco_skills.json _ESCO_SOFT_IDX = set() # int indices of transversal/soft ESCO skills # Optional binary "is this label soft?" classifier head trained on top of # JobBERT-v2. Loaded lazily; if absent, runtime falls back to rule-based # routing (denylist + ESCO transversal lookup). _SOFTNESS_CLF = None _SOFTNESS_THRESHOLD = 0.7 # conservative — only fire on confident soft hits # LLM-classified overrides for every skill seen in resume_data.csv. Produced # by llm_classify_skills.py. Authoritative when present — overrides denylist # and ESCO transversal lookup. Maps lowercase label → {"category", "...}. _SKILL_OVERRIDES: dict[str, dict] = {} def _load_esco_index() -> None: """Best-effort load of the ESCO embedding cache. Idempotent.""" global _ESCO_OK, _ESCO_MODEL, _ESCO_EMBEDS, _ESCO_META, _ESCO_SOFT_IDX import numpy as _np _model_dir = os.path.join(config.BASE_DIR, "models") _embed_path = os.path.join(_model_dir, "esco_embeddings.npy") _labels_path = os.path.join(_model_dir, "esco_skills.json") if not (os.path.exists(_embed_path) and os.path.exists(_labels_path)): return try: from sentence_transformers import SentenceTransformer _ESCO_EMBEDS = _np.load(_embed_path) with open(_labels_path, encoding="utf-8") as f: _ESCO_META = json.load(f) model_name = _ESCO_META.get("model", "TechWolf/JobBERT-v2") _ESCO_MODEL = SentenceTransformer(model_name) _ESCO_OK = True print(f"[Extractor] ESCO index loaded — " f"{_ESCO_EMBEDS.shape[0]:,} skills via {model_name}") # Map transversal-skill conceptUris to indices into _ESCO_EMBEDS so we # can route soft hits at query time without rebuilding the index. _transversal_path = os.path.join(config.BASE_DIR, "data", "esco_transversal_uris.txt") if os.path.exists(_transversal_path): soft_uris = set() with open(_transversal_path, encoding="utf-8") as f: for line in f: line = line.strip() if not line or line.startswith("#"): continue soft_uris.add(line.split("\t", 1)[0]) uris = _ESCO_META.get("conceptUri", []) _ESCO_SOFT_IDX = {i for i, u in enumerate(uris) if u in soft_uris} print(f"[Extractor] Transversal index loaded — " f"{len(_ESCO_SOFT_IDX):,} soft-skill URIs") except Exception as e: print(f"[Extractor] ESCO index load failed: {type(e).__name__}: {e}") _ESCO_OK = False _load_esco_index() def _load_softness_classifier() -> None: """Lazy-load the optional JobBERT softness classifier. Idempotent.""" global _SOFTNESS_CLF if _SOFTNESS_CLF is not None: return import pickle _path = os.path.join(config.BASE_DIR, "models", "softness_classifier.pkl") if not os.path.exists(_path): return try: with open(_path, "rb") as f: _SOFTNESS_CLF = pickle.load(f) m = _SOFTNESS_CLF.get("metrics", {}) print(f"[Extractor] Softness classifier loaded — " f"F1(soft)={m.get('f1_soft', 0):.3f}, " f"threshold={_SOFTNESS_THRESHOLD}") except Exception as e: print(f"[Extractor] Softness classifier load failed: " f"{type(e).__name__}: {e}") _load_softness_classifier() def _load_skill_overrides() -> None: """ Load data/skill_softness_overrides.csv if present. This is the LLM- classified routing table for every skill seen in resume_data.csv — authoritative when present, overrides denylist + ESCO transversal lookup. """ import csv as _csv global _SKILL_OVERRIDES path = os.path.join(config.BASE_DIR, "data", "skill_softness_overrides.csv") if not os.path.exists(path): return try: with open(path, encoding="utf-8") as f: for row in _csv.DictReader(f): lbl = (row.get("label") or "").strip() cat = (row.get("category") or "").strip().lower() if lbl and cat in {"soft", "hard", "language", "other"}: _SKILL_OVERRIDES[lbl.lower()] = { "category": cat, "confidence": (row.get("confidence") or "").strip().lower(), "human_reviewed": (row.get("human_reviewed") or "").strip(), } from collections import Counter as _Counter cats = _Counter(v["category"] for v in _SKILL_OVERRIDES.values()) print(f"[Extractor] Skill overrides loaded — " f"{len(_SKILL_OVERRIDES):,} labels ({dict(cats)})") except Exception as e: print(f"[Extractor] Skill overrides load failed: " f"{type(e).__name__}: {e}") _load_skill_overrides() def _classify_softness(labels: list[str]) -> dict[str, float]: """ Return {label: P(soft)} for each input label. Uses JobBERT-v2 encoder that's already loaded for ESCO. If the classifier isn't available, returns an empty dict (caller falls back to rule-based routing). """ if _SOFTNESS_CLF is None or _ESCO_MODEL is None or not labels: return {} import numpy as _np enc = _ESCO_MODEL.encode( labels, normalize_embeddings=True, show_progress_bar=False, convert_to_numpy=True, ).astype(_np.float32, copy=False) proba = _SOFTNESS_CLF["model"].predict_proba(enc)[:, 1] return {labels[i]: float(proba[i]) for i in range(len(labels))} def _normalise_skill_label(label: str) -> str: """ ESCO often disambiguates skills with parenthetical context, e.g. 'Python (programming language)'. Strip these so the resulting label keyword-matches whatever a JD's required_skills list contains. """ out = re.sub(r"\s*\([^)]+\)\s*", " ", label).strip() return re.sub(r"\s+", " ", out) def _chunk_resume_text(text: str, max_chunks: int = 40) -> list: """ Split a resume into short, embedding-friendly chunks. We DON'T pull in nltk just for this: a simple regex split on sentence-end punctuation + line breaks works fine for resume text, which is mostly bullets and short sentences. """ parts = re.split(r"(?<=[\.!?])\s+|\n+|•|·|■|►|▪|◦", text or "") chunks = [] for p in parts: s = p.strip() if 12 <= len(s) <= 280: chunks.append(s) if len(chunks) >= max_chunks: break return chunks def _esco_extract_skills(text: str, threshold: float = 0.55, top_k: int = 35) -> tuple[list, list]: """ Return (hard_skills, soft_skills) — ESCO labels semantically present in `text`, split by transversal-skill classification. Algorithm: 1. Split resume into ≤40 short chunks. 2. Encode chunks with JobBERT-v2 → (C, dim) L2-normalised matrix. 3. Dot-product with cached ESCO matrix → (C, N) cosine similarities. 4. For each skill, take the **max** similarity across chunks. 5. Keep top-k whose max-sim ≥ threshold; route by transversal flag. """ if not _ESCO_OK or not text: return [], [] chunks = _chunk_resume_text(text) if not chunks: return [], [] import numpy as _np q = _ESCO_MODEL.encode( chunks, normalize_embeddings=True, show_progress_bar=False, convert_to_numpy=True, ).astype(_np.float32, copy=False) sims = q @ _ESCO_EMBEDS.T # (chunks, N) per_skl = sims.max(axis=0) # best chunk-match per skill keep = _np.argsort(per_skl)[::-1][: top_k * 3] # over-pull, filter labels = _ESCO_META["labels"] hard: list[str] = [] soft: list[str] = [] seen: set[str] = set() for idx in keep: score = float(per_skl[idx]) if score < threshold: break # sorted desc → rest are below threshold label = _normalise_skill_label(labels[idx]) key = label.lower() if key in seen: continue seen.add(key) if int(idx) in _ESCO_SOFT_IDX: soft.append(label) else: hard.append(label) if len(hard) + len(soft) >= top_k: break return hard, soft # ============================================================================= # LOAD SKILLS TAXONOMY # ============================================================================= def _load_taxonomy() -> list: """ Load master skills list from data/skills_taxonomy.txt. Returns lowercase list. Falls back to a built-in starter list if the file doesn't exist yet. """ if os.path.exists(config.SKILLS_TAXONOMY_PATH): with open(config.SKILLS_TAXONOMY_PATH, "r", encoding="utf-8") as f: skills = [line.strip().lower() for line in f if line.strip()] return skills # Built-in starter taxonomy — covers most IT roles # Add more to data/skills_taxonomy.txt to extend this return [ # Languages "python", "java", "javascript", "typescript", "c++", "c#", "golang", "ruby", "php", "swift", "kotlin", "scala", "r", "matlab", "perl", # Web "react", "angular", "vue", "node.js", "django", "flask", "fastapi", "spring boot", "html", "css", "rest api", "graphql", # Data & ML "sql", "mysql", "postgresql", "mongodb", "redis", "elasticsearch", "pandas", "numpy", "scikit-learn", "tensorflow", "pytorch", "keras", "machine learning", "deep learning", "nlp", "computer vision", "data analysis", "data science", "statistics", "tableau", "power bi", "excel", "spark", "hadoop", "hive", # Cloud & DevOps "aws", "azure", "gcp", "docker", "kubernetes", "jenkins", "git", "github", "gitlab", "ci/cd", "terraform", "ansible", "linux", # Other Tech "agile", "scrum", "jira", "confluence", "microservices", "kafka", "airflow", "dbt", "snowflake", "databricks", # HR / Business (for non-tech JDs) "sap", "workday", "salesforce", "crm", "erp", "hris", ] _SKILLS_TAXONOMY = _load_taxonomy() # Soft skills vocabulary for pattern matching _SOFT_SKILLS = [ "communication", "leadership", "teamwork", "collaboration", "problem solving", "critical thinking", "time management", "adaptability", "creativity", "stakeholder management", "presentation", "negotiation", "mentoring", "conflict resolution", "decision making", "project management", "analytical", "detail oriented", "self motivated", "multitasking", ] # Degree keywords for education extraction _DEGREE_KEYWORDS = [ "b.tech", "b.e.", "b.sc", "bsc", "btech", "bachelor", "b.a", "ba", "m.tech", "m.e.", "m.sc", "msc", "mtech", "master", "m.a", "ma", "mba", "ph.d", "phd", "doctorate", "diploma", "pgdm", "pgpm", ] # ============================================================================= # 1. CONTACT INFORMATION # ============================================================================= # ── Indian city / state database for location fallback ─────────────────────── # Used when the LLM returns an empty location field. # Source: India Census 2011 + IT-hub common names. _INDIAN_CITIES = [ # Tier-1 metros "Mumbai", "Delhi", "Bangalore", "Bengaluru", "Hyderabad", "Chennai", "Kolkata", "Pune", "Ahmedabad", "Surat", # Tier-2 "Jaipur", "Lucknow", "Kanpur", "Nagpur", "Indore", "Thane", "Bhopal", "Visakhapatnam", "Vizag", "Patna", "Vadodara", "Ludhiana", "Agra", "Nashik", "Faridabad", "Meerut", "Rajkot", "Varanasi", "Prayagraj", "Allahabad", "Guwahati", "Dehradun", "Jammu", "Aurangabad", # NCR / satellite cities "Noida", "Gurgaon", "Gurugram", "Ghaziabad", "Greater Noida", "Navi Mumbai", # IT hubs & regional capitals "Chandigarh", "Coimbatore", "Kochi", "Cochin", "Trivandrum", "Thiruvananthapuram", "Mysore", "Mysuru", "Mangalore", "Mangaluru", "Hubballi", "Hubli", "Vijayawada", "Madurai", "Bhubaneswar", "Cuttack", "Ranchi", "Raipur", "Jabalpur", "Gwalior", "Jodhpur", "Udaipur", "Kota", "Amritsar", "Jalandhar", "Dhanbad", "Srinagar", ] _INDIAN_STATES = [ "Maharashtra", "Karnataka", "Tamil Nadu", "Andhra Pradesh", "Telangana", "Gujarat", "Rajasthan", "Uttar Pradesh", "West Bengal", "Madhya Pradesh", "Kerala", "Punjab", "Haryana", "Bihar", "Odisha", "Jharkhand", "Assam", "Delhi", "NCR", "NCT", ] def _extract_location_from_text(text: str) -> str: """ Best-effort location extraction from raw resume text. Strategy: 1. 'City, State' / 'City - State' pattern with known Indian states. 2. Known city name in the first ~350 chars (header zone). 3. Full-text scan up to 600 chars. Returns a city/location string, or "" if nothing found. """ # Pattern 1 — "City, State" with a recognised Indian state state_pat = '|'.join(re.escape(s) for s in _INDIAN_STATES) m = re.search( rf'([A-Z][a-zA-Z\s]{{2,25}})[,\s\-]+(?:{state_pat})\b', text ) if m: # Return the full match (e.g. "Noida, UP" or "Mumbai, Maharashtra") return m.group(0).strip().rstrip(',').strip() # Pattern 2 — header zone (first 350 chars) header = text[:350] for city in _INDIAN_CITIES: if re.search(r'\b' + re.escape(city) + r'\b', header, re.IGNORECASE): # Try to capture a trailing state/region on the same line city_m = re.search( r'\b' + re.escape(city) + r'\b[,\s]*([^\n]{0,30})', header, re.IGNORECASE ) if city_m: extra = city_m.group(1).strip().rstrip(',').strip() extra_first_line = extra.split('\n')[0].strip() if any(s.lower() in extra_first_line.lower() for s in _INDIAN_STATES): return f"{city.title()}, {extra_first_line}" return city.title() # Pattern 3 — full text scan (slower, last resort) for city in _INDIAN_CITIES: if re.search(r'\b' + re.escape(city) + r'\b', text[:600], re.IGNORECASE): return city.title() return "" # ── Post-processing scrubs for LLM hallucinations ──────────────────────────── # Built once at module load — cheap lower-case lookups. _LOCATION_TOKENS = ( {c.lower() for c in _INDIAN_CITIES} | {s.lower() for s in _INDIAN_STATES} | {"ncr", "india", "region", "area", "north", "south", "east", "west"} ) # Pure business-function / department / section-header tokens. # A "company" composed ONLY of these is a section header, not a real employer. _FUNCTION_TOKENS = { # Departments / functions "operations", "management", "marketing", "sales", "finance", "accounting", "administration", "administrative", "hr", "recruitment", "talent", "business", "analysis", "analytics", "strategy", "development", "planning", "credit", "risk", "compliance", "audit", "treasury", "regulatory", "support", "services", "service", "customer", "client", "relationship", "insurance", "banking", "investment", "wealth", "private", "retail", "corporate", "commercial", "consumer", "product", "project", "program", "delivery", "engineering", "production", "manufacturing", "supply", "chain", "logistics", "procurement", "quality", "assurance", "control", "controls", "technology", "information", "data", "research", "transformation", "digital", "general", "cash", "automation", "platform", "process", "processes", "digitization", "fraud", "financial", "leadership", "leadership", "revenue", "pricing", "approval", "approvals", "review", "reviewed", "approval", "growth", "performance", # Industries / domains often mistaken for companies "pension", "life", "health", "asset", "lending", "loans", "mortgage", "payments", "fintech", "ecommerce", "e-commerce", # Acronyms & metrics frequently mis-extracted as company names "kpi", "kpis", "api", "apis", "roi", "sla", "kra", "kras", "bpo", "kpo", "p&l", "pnl", "ai", "ml", "saas", # UI/UX/IT abbreviations sometimes returned as "company" "ui", "ux", "it", # Country / region tokens (frequently leaked from section headers like "UK Life") "uk", "us", "usa", "uae", "europe", "asia", "apac", "emea", "global", # Degree leaks "b.tech", "btech", "m.tech", "mtech", "b.sc", "bsc", "m.sc", "msc", "mba", "ba", "ma", "phd", "pgdm", # Resume section-header words (LLM sometimes returns these as "companies") "profile", "summary", "core", "competencies", "competency", "professional", "work", "history", "objective", "overview", "highlights", "expertise", "achievements", "awards", "education", "documentation", "contact", "address", "id", "email", # Product / industry descriptor words (not a company name on their own) "personal", "durables", "consumer", "sme", "msme", # Finance / compliance acronyms often leaked from job descriptions "kyc", "aml", "emi", "npa", "nro", "nre", "upi", "pos", "nbfc", "mfi", "rbi", "irda", "amfi", "sebi", "lap", "tprm", # Banking / NBFC function words commonly leaked as fake company names "account", "accounts", "key", "vendor", "payment", "payments", "escrow", "nodal", "loan", "loans", "detection", "collection", "collections", "people", "stakeholder", "stakeholders", "limit", "loading", "borrower", "disbursement", "reconciliation", "portfolio", "bucket", "delinquency", "finops", "fin-ops", "housing", "home", "projects", "sourcing", "monitoring", "team", "building", "performance", "legal", "linkedin", "inr", "usd", "pre", "paid", "instrument", "instruments", # Job-title seniority/level words — appear in role prefixes, not company names "head", "vice", "vp", "avp", "svp", "evp", } # Job-title prefixes that the LLM sometimes pastes in front of a company name # e.g. "Country Head – League of Extraordinary" → starts with "Country Head" # Any company-string that starts with one of these prefixes is a role label, not a company. _JOB_TITLE_PREFIX_RE = re.compile( r'^(?:' r'country\s+head|head\s+of|vp\s+of|vice\s+president(?:\s+of)?|' r'avp\s+(?:of\s+)?|svp\s+(?:of\s+)?|evp\s+(?:of\s+)?|' r'chief\s+(?:executive|operating|financial|technology|marketing|people)|' r'managing\s+director|general\s+manager|deputy\s+(?:general\s+)?manager|' r'senior\s+vice\s+president|assistant\s+vice\s+president|' r'regional\s+(?:head|manager|director)|national\s+(?:head|manager|director)|' r'zonal\s+(?:head|manager|director)|business\s+head|' r'president\s+(?:and|&)\s+|founder\s+(?:and|&)\s+|' r'cluster\s+head|branch\s+head|area\s+(?:head|manager)' r')\b', re.IGNORECASE, ) # Corporate suffixes — any of these in the string strongly suggests a real company _CORPORATE_SUFFIXES = { "ltd", "ltd.", "limited", "pvt", "pvt.", "private", "inc", "inc.", "incorporated", "corp", "corp.", "corporation", "llc", "llp", "plc", "gmbh", "ag", "sa", "co", "co.", "company", "bank", "industries", "technologies", "solutions", "systems", "services", "consulting", "consultants", "enterprises", "group", "holdings", "international", } def _looks_like_location(text: str) -> bool: """ True iff `text` is essentially a place name (no real company-name signal). Used to reject hallucinated 'last_company' values like 'Delhi-NCR'. """ if not text: return False t = text.lower().strip(' .,-–—()') if not t: return False tokens = [w for w in re.split(r"[\s,\-–—/&]+", t) if w] if not tokens: return False connectors = {"and", "the", "of", "in", "at"} return all((tok in _LOCATION_TOKENS or tok in connectors) for tok in tokens) def _looks_like_function(text: str) -> bool: """ True iff `text` is purely a job/department/function name with no company- specific proper-noun tokens. E.g. 'Insurance Operations', 'Credit Management', 'Bank', 'FCA & PRA Compliance'. Decision rule: a real company must have at least one PROPER NOUN — i.e. a token that is NOT in any of (function, location, regulator, connector, corporate-suffix) sets. Otherwise it's just a description. """ if not text: return False t = re.sub(r"\s+", " ", text.lower()).strip() if not t: return False tokens = [w.strip(".,") for w in re.split(r"[\s/&\-–—,]+", t) if w] tokens = [tok for tok in tokens if tok] if not tokens: return False connectors = {"and", "the", "of", "in", "at", "for"} # Regulator / non-company acronyms (UK Financial Conduct Authority, etc.) regulators = { "fca", "pra", "sec", "rbi", "icai", "fda", "hipaa", "sox", "gdpr", "pci", "irda", "sebi", "nse", "bse", } # A proper-noun token = not in any of the "common / generic" pools proper_noun_tokens = [ tok for tok in tokens if tok not in _FUNCTION_TOKENS and tok not in _LOCATION_TOKENS and tok not in _CORPORATE_SUFFIXES and tok not in connectors and tok not in regulators ] # No proper noun → it's just function/location/suffix words → reject if not proper_noun_tokens: return True return False def _looks_like_garbage(text: str) -> bool: """ True iff `text` is obviously not a company name — bullet text, all punctuation, too short, oversized fragments, or PDF unicode artifacts. """ if not text: return True t = text.strip() # Contains a newline → multi-line section fragment, not a single company if '\n' in t or '\r' in t: return True # Absurdly long → entire section header / bullet cluster if len(t) > 80: return True # Starts with a bullet/symbol or unicode dash → bullet text fragment # Includes ASCII hyphen, em-dash (—), en-dash (–), minus (−) if re.match(r'^[•●○◦►▪▫◆◇■□★☆\-\*\+§¶—–−]', t): return True # PDF unicode private-use area artifacts (Wingdings glyphs etc.) # These appear as , , … when PDF bullets are extracted if re.match(r'^[-]', t): return True # Anywhere in the first 3 chars if any(0xf000 <= ord(ch) <= 0xf0ff for ch in t[:3]): return True # Has no alphabetic content at all if not re.search(r'[A-Za-z]', t): return True # Strip non-letter chars; if ≤ 2 letters remain → garbage # (catches "UI", "P&L", "HR", "IT" etc.) letters_only = re.sub(r'[^A-Za-z]', '', t) if len(letters_only) <= 2: return True # Contains internal bullet markers → multi-bullet fragment if re.search(r'[•●◦►▪▫◆◇■□]', t): return True # Unbalanced parentheses → truncated/corrupted string # e.g. "Anti, Money Laundering (AML" or "FIU IND (CIMS & DAKSH" if t.count('(') != t.count(')'): return True # Contains '@' → email address leaked as company name if '@' in t: return True # Starts with a job-title prefix → role label, not a company name # e.g. "Country Head League of Extraordinary", "Head of Insurance" if _JOB_TITLE_PREFIX_RE.match(t): return True # Suspicious starts — verb or structural phrase that identifies a non-company string suspicious_starts = ( 'executed ', 'facilitated ', 'currently', 'working with ', 'handled ', 'guided ', 'collaborated ', 'monitored ', 'led ', 'managed ', 'developed ', 'created ', 'designed ', 'built ', 'implemented ', 'delivered ', 'achieved ', 'worked ', 'responsible ', 'role ', 'duty ', 'reviewed ', 'approved ', 'drove ', 'utilize ', 'utilized ', 'utilise ', 'utilised ', 'directed ', 'oversaw ', 'spearheaded ', 'ensured ', 'supported ', 'assisted ', 'coordinated ', 'analyzed ', 'analysed ', ) if t.lower().startswith(suspicious_starts): return True return False def _clean_company_name(co: str) -> str: """ Best-effort cleanup of an LLM-emitted company name. Rejects: - Bullet text fragments ("• Executed daily reviews") - Pure locations ("Delhi NCR") - Pure job functions ("Insurance Operations", "Credit Management") - Punctuation-only or too-short strings Keeps: - 'Infosys' → 'Infosys' - 'HDFC Bank, Mumbai' → 'HDFC Bank' - 'Groz Engineering Tools Pvt. Ltd.' → 'Groz Engineering Tools Pvt. Ltd.' - 'Yes Bank Ltd' → 'Yes Bank Ltd' """ if not co: return "" raw = co.strip() if _looks_like_garbage(raw): return "" if _looks_like_location(raw): return "" if _looks_like_function(raw): return "" # Strip trailing location markers via segment split parts = re.split(r"\s*[,\-–—]\s*", raw) kept = [] for p in parts: p = p.strip() if not p: continue if _looks_like_location(p): continue if _looks_like_function(p): continue kept.append(p) cleaned = ", ".join(kept) if kept else "" # Final sanity check after splitting if not cleaned or _looks_like_garbage(cleaned): return "" return cleaned.strip() def _filter_project_entries(projects: list, companies: list) -> list: """ Remove entries that look like company names or pure work-history bullets from the projects list. The LLM sometimes leaks employer names here. """ if not projects: return [] co_keys = {c.lower().strip() for c in companies if c} cleaned = [] for p in projects: if not p or not p.strip(): continue ps = p.strip() # Drop direct company-name matches if ps.lower() in co_keys: continue # Drop bare location strings if _looks_like_location(ps): continue # Drop entries whose leading token is a known company name. # Splits on | — – - , ( so "HDFC Bank, Mumbai (2018-2022)" → "HDFC Bank" first_part = re.split(r'[|—–\-,(]', ps, 1)[0].strip() if first_part and first_part.lower() in co_keys: continue # Drop entries whose first two words match a company prefix # e.g. "HDFC Bank quarterly review" when company is "HDFC Bank" first_two = " ".join(first_part.split()[:2]).lower() if first_two and first_two in co_keys: continue cleaned.append(ps) return cleaned def _years_from_tenures(tenures: list) -> float: """ Sum non-overlapping calendar time across a list of company_tenures entries. "Present"/"Current"/"Now" are anchored to today's date — NOT the date the CV was written. Returns total years (rounded to 1 dp), or 0 if no parseable date ranges were found. Accepts both formats: - "Company | Role | Start – End" (new format) - "Company – City (Start – End)" (legacy format) """ if not tenures: return 0.0 _month = ( r'(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|' r'Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|' r'Nov(?:ember)?|Dec(?:ember)?)' ) range_re = re.compile( rf'({_month}\s*\'?\d{{2,4}}|20\d{{2}})\s*[-–—]\s*' rf'({_month}\s*\'?\d{{2,4}}|20\d{{2}}|[Pp]resent|[Cc]urrent|[Nn]ow|[Oo]ngoing)', re.IGNORECASE, ) now = datetime.now() pairs: list = [] for t in tenures: m = range_re.search(str(t)) if not m: continue start_str, end_str = m.group(1).strip(), m.group(2).strip().lower() start = _parse_date_safe(start_str) end = now if end_str in ('present', 'current', 'now', 'ongoing') \ else _parse_date_safe(end_str) if start and end and end >= start: pairs.append((start, end)) if not pairs: return 0.0 merged_months = 0 for s, e in _merge_date_ranges(pairs): d = relativedelta(e, s) merged_months += d.years * 12 + d.months return round(merged_months / 12, 1) def _build_tenures_from_text(text: str, companies: list) -> list: """ Regex fallback: build "Company – City (Month Year – Month Year)" entries by scanning the raw resume text for date ranges near each known company name. Called only when the LLM returns an empty company_tenures list. Strategy per company: 1. Find the company name's first occurrence in the text. 2. In the next ~400 chars search for a month-year date range. 3. Optionally capture a known Indian city in the same window. 4. Emit the formatted tenure string. """ if not companies: return [] # Pattern A — "Month Year – Month Year" (most common in structured CVs) _month = ( r'(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|' r'Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|' r'Nov(?:ember)?|Dec(?:ember)?)' ) _date_range_re = re.compile( rf'({_month}\s*\'?\d{{2,4}})\s*[-–—]\s*' rf'({_month}\s*\'?\d{{2,4}}|[Pp]resent|[Cc]urrent)', re.IGNORECASE, ) # Pattern B — "Year – Year" or "Year – Present" (year-only ranges) _year_range_re = re.compile( r'(20\d{2})\s*[-–—]\s*(20\d{2}|[Pp]resent|[Cc]urrent)', re.IGNORECASE, ) tenures: list = [] seen: set = set() for company in companies: # Find position of this company in the full text (case-insensitive) try: idx = text.lower().index(company.lower()) except ValueError: continue # 400-char window starting at the company mention window = text[idx: idx + 400] m_month = _date_range_re.search(window) m_year = _year_range_re.search(window) if m_month and m_year: # Take whichever date range appears first (closest to the company) if m_month.start() <= m_year.start(): start_str = m_month.group(1).strip() end_raw = m_month.group(2).strip() else: start_str = m_year.group(1).strip() end_raw = m_year.group(2).strip() elif m_month: start_str = m_month.group(1).strip() end_raw = m_month.group(2).strip() elif m_year: start_str = m_year.group(1).strip() end_raw = m_year.group(2).strip() else: continue # Normalise "present"/"current" → "Present" end_str = "Present" if end_raw.lower() in ("present", "current") else end_raw # Look for a known Indian city in the first 200 chars of the window city = "" for cname in _INDIAN_CITIES: if re.search(r'\b' + re.escape(cname) + r'\b', window[:200], re.IGNORECASE): city = cname.title() break dates = f"{start_str} – {end_str}" key = (company.lower(), dates.lower()) if key in seen: continue seen.add(key) if city: tenures.append(f"{company} – {city} ({dates})") else: tenures.append(f"{company} ({dates})") return tenures _NAME_SKIP_WORDS = { # Document type / section labels that bubble up into the resume header 'resume', 'cv', 'curriculum', 'vitae', 'profile', 'objective', 'summary', 'contact', 'information', 'details', 'education', 'experience', 'skills', 'certifications', 'projects', 'hobbies', 'interests', 'achievements', 'career', 'personal', 'professional', # Job-title nouns — common false positives when scanning header lines 'senior', 'junior', 'lead', 'manager', 'analyst', 'engineer', 'developer', 'consultant', 'specialist', 'executive', 'director', 'intern', 'trainee', 'associate', 'accountant', 'designer', 'copywriter', 'writer', 'marketer', 'page', 'of', # Address / contact noise 'mobile', 'phone', 'email', 'address', 'linkedin', 'github', 'india', 'delhi', 'mumbai', 'bangalore', 'pune', 'hyderabad', } # Explicit labels that prefix a name on the resume header _NAME_LABEL_PATTERNS = [ # "Name: John Doe" / "Candidate Name - John Doe" re.compile( r'^\s*(?:name|candidate\s*name|full\s*name|applicant\s*name)\s*[:\-–]\s*' r'([A-Za-z][A-Za-z\.\-\']+(?:\s+[A-Za-z][A-Za-z\.\-\']+){1,3})\s*$', re.IGNORECASE, ), # "I am John Doe" / "My name is John Doe" re.compile( r'^\s*(?:i\s+am|my\s+name\s+is|this\s+is)\s+' r'([A-Za-z][A-Za-z\.\-\']+(?:\s+[A-Za-z][A-Za-z\.\-\']+){1,3})\b', re.IGNORECASE, ), ] # 2–4 ALL-CAPS tokens, optionally separated by " - " — very common header style _NAME_ALLCAPS_PATTERN = re.compile( r'^\s*([A-Z][A-Z\.\-\']{1,}(?:\s+[A-Z][A-Z\.\-\']{1,}){1,3})\s*$' ) # Title-cased "First Last [Middle]" pattern, allowing single-letter middles _NAME_TITLECASE_PATTERN = re.compile( r'^\s*([A-Z][a-z]+(?:\s+[A-Z]\.?)?\s+[A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)\s*$' ) def _looks_like_name(text: str) -> bool: """Lightweight gate used by every regex/NER branch.""" words = text.strip().split() if not (2 <= len(words) <= 4): return False if re.search(r'[0-9@|/\\:_]', text): return False if any(w.lower() in _NAME_SKIP_WORDS for w in words): return False if len(text.strip()) < 4: return False return True def _header_lines(text: str, n: int = 10) -> list: """ Return the first `n` non-empty lines of the resume. Name extraction is restricted to this slice because resume names live in the header. """ out = [] for ln in text.split('\n'): s = ln.strip() if not s: continue out.append(s) if len(out) >= n: break return out def extract_candidate_name(text: str, file_name: str = "") -> str: """ Extract candidate name from a resume. Strategy (in order of reliability): 1. spaCy PERSON entity — but only on the **first 10 non-empty lines**, not the full document. Stops mid-resume "John, my mentor" hits. 2. Regex fallback — explicit "Name: John Doe" labels. 3. Regex fallback — ALL-CAPS header (e.g. "GAURAV JOSHI"). 4. Regex fallback — Title-case "First Last" pattern. 5. Loose line scan inside the header zone. 6. Filename derivation as a last resort. Args: text : Raw resume text file_name : Original filename — used as last-resort fallback Returns: Candidate name string, or "Unknown" if nothing found. """ if not text: return _name_from_filename(file_name) header = _header_lines(text, n=10) header_text = "\n".join(header) # ── Strategy 1: spaCy NER on header zone only ──────────────────────────── if _SPACY_OK and _NLP and header_text: try: doc = _NLP(header_text) for ent in doc.ents: if ent.label_ == "PERSON" and _looks_like_name(ent.text): cleaned = re.sub(r"[^a-zA-Z\s\.\-']", '', ent.text).strip() if cleaned and len(cleaned) >= 4: return cleaned.title() except Exception: pass # spaCy occasionally chokes on garbled OCR text # ── Strategy 2: Regex — explicit "Name:" labels in the header ──────────── for ln in header: for pat in _NAME_LABEL_PATTERNS: m = pat.search(ln) if m and _looks_like_name(m.group(1)): return m.group(1).strip().title() # ── Strategy 3: Regex — ALL-CAPS header line (top 5 lines only) ────────── for ln in header[:5]: m = _NAME_ALLCAPS_PATTERN.match(ln) if m and _looks_like_name(m.group(1)): return m.group(1).strip().title() # ── Strategy 4: Regex — Title-cased "First Last" ───────────────────────── for ln in header[:6]: m = _NAME_TITLECASE_PATTERN.match(ln) if m and _looks_like_name(m.group(1)): return m.group(1).strip().title() # ── Strategy 5: Loose pattern scan over the full header ────────────────── for ln in header: words = ln.split() if (2 <= len(words) <= 4 and not re.search(r'[0-9@|/\\:_\.]', ln) and all(w[0].isupper() for w in words if w) and not any(w.lower() in _NAME_SKIP_WORDS for w in words) and len(ln) >= 5): return ln.title() # ── Strategy 6: Derive from filename ───────────────────────────────────── return _name_from_filename(file_name) def _name_from_filename(file_name: str) -> str: """ Derive a display name from a resume filename. Examples: "priya_sharma.pdf" → "Priya Sharma" "Gaurav_Joshi_Resume pdf.pdf" → "Gaurav Joshi" "Naukri_AayushiShukla[6y_0m].pdf" → "Aayushishukla" "kartik CV.pdf" → "Kartik" "resume_1.pdf" → "Unknown" """ if not file_name: return "Unknown" stem = os.path.splitext(os.path.basename(file_name))[0] # Replace underscores, hyphens, dots with spaces stem = re.sub(r'[_\-\.]+', ' ', stem).strip() # Strip brackets and their contents e.g. [6y_0m] stem = re.sub(r'\[.*?\]|\(.*?\)', ' ', stem).strip() # Remove any remaining non-alphabetic characters from each token words = [re.sub(r'[^a-zA-Z]', '', w) for w in stem.split()] words = [w for w in words if w] # drop empty tokens after stripping # Skip generic filename tokens (including file-extension leftovers) generic = { 'resume', 'cv', 'curriculum', 'vitae', 'application', 'candidate', 'updated', 'final', 'new', 'f', 'v', 'v2', 'v3', 'pdf', 'docx', 'doc', 'txt', 'naukri', 'linkedin', 'profile', 'my', 'latest', 'copy', '1', '2', '3', '2024', '2025', '2026', } filtered = [w for w in words if w.lower() not in generic and len(w) >= 2 and not w.isdigit()] if len(filtered) >= 2: return ' '.join(w.title() for w in filtered[:3]) # Accept single meaningful word as a first-name-only fallback if len(filtered) == 1 and len(filtered[0]) >= 3: return filtered[0].title() return "Unknown" def extract_contact(text: str) -> dict: """ Extract email address and phone number from resume text. Phone extraction — multi-strategy, Indian-mobile-first: 1. +91 prefix → capture the 10 digits that follow (+91 removed as prefix) 2. 91 without + → same 3. Leading 0 → STD-style (0 removed as prefix) 4. Bare 10-digit mobile starting with 6-9 5. Generic international fallback The old ``lstrip('+91')`` was a character-set strip (stripped any of '+','9','1'), which turned '+91 9953668257' → '53668257'. This version uses regex prefix removal so every digit is preserved. Returns: {"email": "x@y.com", "phone": "9876543210"} Values are empty string if not found. """ # Email — standard pattern email_match = re.search( r'[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}', text ) email = email_match.group(0) if email_match else "" phone = "" # Strategy 1: +91 [sep?] 10-digit-mobile (with optional mid-separators) m = re.search(r'\+91[\s\-\.]?([6-9]\d{2})[\s\-\.]?(\d{3})[\s\-\.]?(\d{4})', text) if m: phone = m.group(1) + m.group(2) + m.group(3) # Strategy 2: 91 (no +) then 10-digit mobile — e.g. "919876543210" if not phone: m = re.search(r'(? 10 else digits phone = digits[-10:] if len(digits) >= 10 else digits return {"email": email, "phone": phone} def extract_urls(text: str) -> dict: """ Pull every clickable URL out of the resume and bucket them by type: - linkedin_url : LinkedIn profile (linkedin.com/in/…) - github_url : GitHub profile or repo (github.com/…) - portfolio_url : First non-LinkedIn / non-GitHub personal site (heuristic: looks like a portfolio domain or contains personal keywords) - other_urls : Every remaining URL — kept as a JSON list so the UI can render them as extra link chips. The patterns are loose on protocol — we accept "https://", "http://" or bare "www.linkedin.com/in/foo" because Indian CVs frequently strip the protocol when exporting from Word to PDF. Returns dict with all four keys; values are "" or [] when not found. """ # Canonical URL pattern — accepts protocol-less domains too. url_re = re.compile( r'(?:https?://|www\.)[A-Za-z0-9._~:/?#@!$&\'()*+,;=\-%]+', re.IGNORECASE, ) candidates = [] for m in url_re.finditer(text or ""): u = m.group(0).strip().rstrip('.,;:)]}') if u.lower().startswith("www."): u = "https://" + u if u not in candidates: candidates.append(u) # Also pick up bare LinkedIn slugs even when the user wrote just # "linkedin.com/in/foo" without www/https — common in scraped PDFs. for m in re.finditer( r'(?:^|\s)((?:linkedin\.com|github\.com)/[A-Za-z0-9._~:/?#@!$&\'()*+,;=\-%]+)', text or "", re.IGNORECASE, ): u = "https://" + m.group(1).strip().rstrip('.,;:)]}') if u not in candidates: candidates.append(u) linkedin = "" github = "" portfolio = "" others = [] PORTFOLIO_HINT_RE = re.compile( r'(portfolio|behance|dribbble|medium|substack|notion\.site|' r'github\.io|wordpress|wixsite|webflow|carrd|vercel\.app|' r'netlify\.app|pages\.dev)', re.IGNORECASE, ) SKIP_RE = re.compile( r'(google\.com/maps|wa\.me|tel:|mailto:|youtu\.be|youtube\.com/watch)', re.IGNORECASE, ) for u in candidates: lo = u.lower() if SKIP_RE.search(lo): continue if not linkedin and "linkedin.com" in lo: linkedin = u continue if not github and "github.com" in lo: github = u continue if not portfolio and PORTFOLIO_HINT_RE.search(lo): portfolio = u continue others.append(u) return { "linkedin_url" : linkedin, "github_url" : github, "portfolio_url" : portfolio, "other_urls" : others[:5], # cap at 5 to keep CSV row sane } # ============================================================================= # 2. SKILLS EXTRACTION # ============================================================================= def extract_skills(text: str, taxonomy: list = None, category: str = "", job_title: str = "", esco_threshold: float = 0.55, esco_top_k: int = 35) -> list: """ Extract technical skills from resume / JD text. Layered strategy (each layer adds to a single deduplicated set): 1. ESCO + JobBERT-v2 embedding lookup (Layer 1, recall-oriented). Built via build_esco_index.py. Catches semantic mentions like "built ETL pipelines" → Apache Airflow / Data Engineering even when the literal skill name never appears. 2. Taxonomy keyword match (precision-oriented). Anchors high-confidence exact matches against data/skills_taxonomy.txt — guarantees obvious skills like "Python" or "SQL" never get missed even if ESCO is unavailable. 3. Legacy TF-IDF classifier (train_skill_model.py). Only used when the ESCO index has NOT been built yet, so older deployments keep working until they run build_esco_index.py. Args: text : Raw resume or JD text taxonomy : Override skills list (defaults to loaded taxonomy) category : Job category hint for the legacy ML model job_title : Job title hint for the legacy ML model esco_threshold: Min cosine similarity to keep an ESCO label (0–1) esco_top_k : Max ESCO labels per call Returns: Sorted deduplicated list of skill strings. """ if not text: return [] # Soft-skill block set. Any label whose lowercase form is in here will be # rejected by _add() regardless of which layer produced it — this is what # prevents "Leadership"/"Communication" from leaking out of the taxonomy # match and into technical_skills. _soft_blocklist = {s.lower() for s in _SOFT_SKILLS} if _ESCO_OK and _ESCO_META: _labels = _ESCO_META.get("labels", []) for _idx in _ESCO_SOFT_IDX: if 0 <= _idx < len(_labels): _soft_blocklist.add(_normalise_skill_label(_labels[_idx]).lower()) found = {} # lower-case key → display label (preserves first form seen) def _add(label: str) -> None: if not label: return key = label.lower() if key in _soft_blocklist: return if key not in found: found[key] = label # ── Layer 1: ESCO semantic lookup (hard skills only — soft hits are # routed by transversal flag and consumed by extract_soft_skills) ───────── if _ESCO_OK: hard_hits, _soft_hits = _esco_extract_skills(text, threshold=esco_threshold, top_k=esco_top_k) for s in hard_hits: _add(s) # ── Layer 2: Taxonomy keyword match (precision floor) ──────────────────── tax = taxonomy or _SKILLS_TAXONOMY text_lower = text.lower() for skill in tax: pattern = r'\b' + re.escape(skill) + r'\b' if re.search(pattern, text_lower): display = skill.upper() if len(skill) <= 3 else skill.title() _add(display) # ── Layer 3: Legacy TF-IDF predictions (only if ESCO not yet built) ────── if not _ESCO_OK and _ML_MODEL_OK: for s in _ml_predict(text, category=category, job_title=job_title): _add(s) return sorted(found.values()) def extract_soft_skills(text: str) -> list: """ Extract soft skills from resume text. Sources (deduplicated, case-insensitive): 1. ESCO transversal-skill hits via JobBERT-v2 (data/esco_transversal_uris.txt) 2. The legacy 20-word _SOFT_SKILLS regex denylist """ if not text: return [] found: dict[str, str] = {} # lower-case key → display label def _add(label: str) -> None: if not label: return k = label.lower() if k not in found: found[k] = label # Layer 1: ESCO transversal hits (JobBERT semantic match) if _ESCO_OK: _hard, soft_hits = _esco_extract_skills(text) for s in soft_hits: _add(s) # Layer 2: regex denylist (catches obvious surface forms ESCO may miss) text_lower = text.lower() for skill in _SOFT_SKILLS: if re.search(r'\b' + re.escape(skill) + r'\b', text_lower): _add(skill.title()) return sorted(found.values()) # ============================================================================= # 3. WORK EXPERIENCE # ============================================================================= def _parse_date_safe(date_str: str): """ Safely parse a date string. Returns datetime or None. """ try: return dateparser.parse(date_str, default=datetime(2000, 1, 1)) except Exception: return None def _merge_date_ranges(ranges: list) -> list: """ Given a list of (start_datetime, end_datetime) pairs, sort them by start date and merge any overlapping or adjacent intervals. Example: [(Jan-2020, Jun-2021), (Mar-2021, Dec-2022)] → [(Jan-2020, Dec-2022)] ← correctly avoids double-counting overlap Returns the merged list sorted by start date. """ if not ranges: return [] sorted_r = sorted(ranges, key=lambda x: x[0]) merged = [list(sorted_r[0])] # work with mutable lists for start, end in sorted_r[1:]: prev_start, prev_end = merged[-1] if start <= prev_end: # overlapping or touching merged[-1][1] = max(prev_end, end) else: merged.append([start, end]) return [(s, e) for s, e in merged] # Section-header keyword maps. Used by _build_section_map to label every # character offset in the resume with the section it belongs to, so date # ranges can be classified by SECTION (not by ±N-char window, which bleeds # across section boundaries in dense CVs). _SECTION_HEADERS = { "work": [ "professional experience", "work experience", "employment history", "career history", "experience", "work history", "professional summary", "professional history", "employment", "career", ], "internship": [ "internship", "internships", "internship experience", "summer internship", "summer training", "industrial training", "training", "apprenticeship", "trainings & internships", ], "education": [ "education", "academic", "qualifications", "academics", "educational background", "academic background", "academic qualification", ], "projects": [ "projects", "project", "key projects", "academic projects", "personal projects", "selected projects", ], "certifications": [ "certifications", "certificates", "courses", "credentials", ], "achievements": [ "achievements", "awards", "honors", "honours", "recognitions", "publications", "patents", ], } # Header line pattern — short, mostly-uppercase or title-case, optionally # followed by colon or dash. Real-world CV headers are ≤40 chars. _HEADER_LINE_RE = re.compile( r'^\s*([A-Z][A-Za-z &/]{2,40})\s*[:\-–—]?\s*$' ) def _build_section_map(text: str) -> list: """ Walk the resume line-by-line and emit a list of (char_offset, section_kind) tuples marking where each section starts. `section_kind` is one of "work" / "internship" / "education" / "projects" / "certifications" / "achievements" / "unknown". The list is sorted by offset. To classify a date range, binary-search this list for the last entry whose offset is ≤ the range's start — that entry's kind is the section the range belongs to. Falls back to "unknown" for content before any header. """ sections: list = [(0, "unknown")] offset = 0 for raw_line in text.splitlines(keepends=True): stripped = raw_line.strip() line_len = len(raw_line) m = _HEADER_LINE_RE.match(stripped) if stripped else None if m: label = m.group(1).strip().lower() kind = "unknown" for sect_kind, keywords in _SECTION_HEADERS.items(): # Match if the header IS one of the keywords (allow short # extras like "PROFESSIONAL EXPERIENCE & PROJECTS") if any(label == k or label.startswith(k + " ") or k in label.split() for k in keywords): kind = sect_kind break if kind != "unknown": sections.append((offset, kind)) offset += line_len return sections def _section_at(sections: list, char_offset: int) -> str: """Return the section kind covering this offset (binary-search-friendly).""" last = "unknown" for off, kind in sections: if off <= char_offset: last = kind else: break return last # Conservative fallback regexes — used ONLY when no section header was found # anywhere in the CV (rare; mostly bullet-only PDFs). They look in a small # ±80-char window so we don't pull "Trainee" / "Project" from a neighbouring # section. _INTERNSHIP_INLINE_RE = re.compile( r'\b(intern(?:ship)?s?|summer\s+training|industrial\s+training|' r'apprentice|articled\s+assistant|practicum|co[-\s]?op)\b', re.IGNORECASE, ) _NON_WORK_INLINE_RE = re.compile( r'\b(bachelor|master|b\.?\s?tech|m\.?\s?tech|b\.?\s?e\.?|m\.?\s?e\.?|' r'b\.?\s?sc|m\.?\s?sc|mba|pgdm|ph\.?d|diploma|class\s+(?:x|xii|10|12)|' r'high\s+school|cbse|icse|university|college|institute|school|' r'certification|certificate|coursera|udemy|nptel|edx|hackerrank|' r'final[-\s]?year\s+project|capstone|thesis|publication|patent|' r'hackathon|olympiad)\b', re.IGNORECASE, ) def _classify_date_range(text: str, span_start: int, span_end: int, sections: list) -> str: """ Decide whether a date range belongs to a full-time role, an internship, or neither (education / project / award). Strategy: 1. Look up which SECTION the range falls into using the pre-computed section map. Sections are derived from real header lines, so this is dramatically more accurate than a sliding window. 2. If the section is "unknown" (no header found, or content before the first header), fall back to a tight inline keyword window. Returns one of: "work" | "internship" | "non_work" """ sect = _section_at(sections, span_start) if sect == "internship": return "internship" if sect in ("education", "projects", "certifications", "achievements"): return "non_work" if sect == "work": return "work" # Unknown section — use a tight inline window lo = max(0, span_start - 80) hi = min(len(text), span_end + 80) window = text[lo:hi] if _NON_WORK_INLINE_RE.search(window): return "non_work" if _INTERNSHIP_INLINE_RE.search(window): return "internship" return "work" # Master date-range regex — captures ALL of: # Jan 2020 – Mar 2023 / January 2020 to March 2023 # Jan'20 – Mar'23 / 01/2020 – 03/2023 # 01-2020 – 03-2023 / 2020 – 2023 # May 2021 – Present / Jun 2022 – Current / Aug 2024 – Ongoing _MONTH_TOKEN = ( r'(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|' r'Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|' r'Nov(?:ember)?|Dec(?:ember)?)' ) _DATE_START_TOKEN = ( rf'(?:{_MONTH_TOKEN}[\s,\.-]*\'?\d{{2,4}}' # Jan 2020 / Jan'20 rf'|(?:0?[1-9]|1[0-2])[\s/.\-](?:19|20)\d{{2}}' # 01/2020 rf'|(?:19|20)\d{{2}})' # 2020 ) _DATE_END_TOKEN = ( rf'(?:{_MONTH_TOKEN}[\s,\.-]*\'?\d{{2,4}}' rf'|(?:0?[1-9]|1[0-2])[\s/.\-](?:19|20)\d{{2}}' rf'|(?:19|20)\d{{2}}' rf'|[Pp]resent|[Cc]urrent|[Nn]ow|[Tt]ill\s+[Dd]ate|[Oo]ngoing)' ) _DATE_RANGE_RE = re.compile( rf'({_DATE_START_TOKEN})\s*(?:[-–—]|to|until|till)\s*({_DATE_END_TOKEN})', re.IGNORECASE, ) def _scan_date_ranges(text: str) -> list: """ Scan the ENTIRE resume text and return a list of dicts: {start: datetime, end: datetime, kind: "work"|"internship"|"non_work", span: (lo, hi)} Each date range is classified by the SECTION header it falls under (Experience, Internship, Education, Projects, Certifications, …) — not by a sliding text window — so section boundaries are respected even when adjacent sections are physically close on the page. Date ranges shorter than 1 month or longer than 50 years are discarded as parser noise. Duplicates (same month/year start AND end) are kept only once so the same line being matched twice doesn't double-count. """ now = datetime.now() sections = _build_section_map(text) out = [] seen = set() for m in _DATE_RANGE_RE.finditer(text): start_str, end_str = m.group(1).strip(), m.group(2).strip() start = _parse_date_safe(start_str) if not start: continue if end_str.lower() in ('present', 'current', 'now', 'till date', 'ongoing'): end = now else: end = _parse_date_safe(end_str) if not end or end < start: continue delta = relativedelta(end, start) months = delta.years * 12 + delta.months if not (0 < months < 600): continue key = (start.year, start.month, end.year, end.month) if key in seen: continue seen.add(key) kind = _classify_date_range(text, m.start(), m.end(), sections) out.append({"start": start, "end": end, "kind": kind, "span": (m.start(), m.end())}) return out def _sum_months(pairs: list) -> int: """Sum non-overlapping calendar months across (start, end) pairs.""" merged = _merge_date_ranges(pairs) total = 0 for s, e in merged: d = relativedelta(e, s) total += d.years * 12 + d.months return total def extract_experience(text: str) -> dict: """ Extract work-experience details from FULL resume text. Strategy (no length cap — scans every character of the input): 1. spaCy ORG entities for company names (kept short-context for speed). 2. Regex sweep for ALL date ranges in the text, including those buried several pages deep. 3. Each date range is classified by surrounding context as "work" / "internship" / "non_work" (education, project, award, etc.) using ±180-char keyword windows. This is what lets us separate the "full-time experience" total from "internship experience" cleanly. 4. Total work experience = SUM of non-overlapping merged work ranges (so two overlapping roles don't double-count), with internship and non-work ranges excluded entirely. Returns: { "total_years" : 4.5, "internship_years" : 0.5, "job_titles" : [...], "companies" : [...], "avg_tenure_months" : 22.0, "last_company" : "TCS", "stability_flag" : False, } """ result = { "total_years" : 0.0, "internship_years" : 0.0, "job_titles" : [], "companies" : [], "avg_tenure_months" : 0.0, "last_company" : "", "stability_flag" : False, } # --- Extract company names via spaCy ORG entities --- # Companies often appear in the first 1-2 pages, but we scan the first # 6000 chars (≈2-3 pages) to balance speed and completeness. The LLM # extractor sees the full text and remains the primary source. companies = [] if _SPACY_OK and _NLP: doc = _NLP(text[:6000]) for ent in doc.ents: if ent.label_ == "ORG": name = ent.text.strip() if (len(name) >= 2 and not name.isdigit() and name not in companies): companies.append(name) result["companies"] = companies[:10] result["last_company"] = companies[0] if companies else "" # --- Extract job titles via common patterns --- title_patterns = [ r'(?i)(senior|junior|lead|principal|associate|staff)?\s*' r'(software|data|ml|ai|backend|frontend|full.?stack|devops|cloud|' r'product|project|hr|talent|recruitment|business|marketing|sales|' r'finance|operations|' r'analyst|engineer|developer|architect|manager|consultant|' r'scientist|specialist|coordinator|executive|intern|trainee)', ] titles = [] for pattern in title_patterns: matches = re.findall(pattern, text) for match in matches: title = ' '.join(m for m in match if m).strip().title() if title and title not in titles and len(title) > 4: titles.append(title) result["job_titles"] = titles[:8] # --- Scan ALL date ranges from the full text and classify each one --- all_ranges = _scan_date_ranges(text) work_pairs = [(r["start"], r["end"]) for r in all_ranges if r["kind"] == "work"] intern_pairs = [(r["start"], r["end"]) for r in all_ranges if r["kind"] == "internship"] if work_pairs: work_months = _sum_months(work_pairs) individual_months = [] for s, e in work_pairs: d = relativedelta(e, s) m = d.years * 12 + d.months if 0 < m < 600: individual_months.append(m) avg_months = (sum(individual_months) / len(individual_months) if individual_months else 0) result["total_years"] = round(work_months / 12, 1) result["avg_tenure_months"] = round(avg_months, 1) result["stability_flag"] = avg_months < config.MIN_TENURE_MONTHS elif not all_ranges: # No date ranges anywhere — last-ditch "X years experience" hint yr_match = re.search( r'(\d+(?:\.\d+)?)\s*\+?\s*years?\s*(?:of\s*)?(?:experience|exp)', text, re.IGNORECASE, ) if yr_match: result["total_years"] = float(yr_match.group(1)) if intern_pairs: result["internship_years"] = round(_sum_months(intern_pairs) / 12, 1) return result # ============================================================================= # 4. EDUCATION # ============================================================================= def extract_education(text: str) -> dict: """ Extract highest education qualification from resume text. Returns: {"degree": "B.Tech", "field": "Computer Science", "institution": "VIT"} """ result = {"degree": "", "field": "", "institution": ""} text_lower = text.lower() # Find degree for deg in sorted(_DEGREE_KEYWORDS, key=len, reverse=True): if re.search(r'\b' + re.escape(deg) + r'\b', text_lower): result["degree"] = deg.upper() if len(deg) <= 5 else deg.title() break # Find field of study — common patterns after degree keyword field_match = re.search( r'(?:b\.?tech|m\.?tech|b\.?sc|m\.?sc|bachelor|master|mba|pgdm)' r'[,\s]+(?:in\s+)?([a-zA-Z\s&/]{3,40}?)(?:\n|,|\.|from|at|–|-)', text, re.IGNORECASE ) if field_match: result["field"] = field_match.group(1).strip().title() # Find institution via spaCy ORG or keyword pattern inst_match = re.search( r'\b(?:university|college|institute|iit\s?\w*|nit\s?\w*|bits|iim|vit|' r'symbiosis|amity|manipal|lpu)\b[a-zA-Z\s,\.]{0,40}', text, re.IGNORECASE ) if inst_match: result["institution"] = inst_match.group(0).strip().title() elif _SPACY_OK and _NLP: # Find ORG entities near degree keywords edu_section = _find_section(text, ["education", "academic", "qualification"]) if edu_section: doc = _NLP(edu_section[:500]) for ent in doc.ents: if ent.label_ == "ORG" and len(ent.text) > 3: result["institution"] = ent.text.strip().title() break return result # ============================================================================= # 5. PROJECTS, CERTIFICATIONS, HOBBIES # ============================================================================= def extract_projects(text: str) -> list: """ Extract project names/descriptions from the Projects section. Returns: List of project name strings (up to 8). """ section = _find_section(text, ["projects", "project work", "key projects"]) if not section: return [] # Each project usually starts with a title on its own line or after a bullet projects = [] lines = section.split('\n') for line in lines: line = line.strip().lstrip('•–-*►▪').strip() # Project line: reasonably short, has uppercase, not just a date if (5 < len(line) < 100 and not re.match(r'^\d{4}', line) and any(c.isupper() for c in line)): projects.append(line) return projects[:8] def extract_certifications(text: str) -> list: """ Extract certifications from resume text. Returns: List of certification name strings (up to 10). """ section = _find_section(text, ["certifications", "certificates", "courses", "training", "achievements"]) source = section if section else text # Common certification keywords cert_patterns = [ r'(?:AWS|Azure|GCP|Google|Microsoft|Oracle|Cisco|PMI|PMP|' r'Scrum|SAFe|ITIL|CompTIA|Coursera|Udemy|edX|NPTEL|' r'HackerRank|HackerEarth|Certified|Certification)' r'[a-zA-Z\s\-:®™]{0,60}', ] certs = [] for pattern in cert_patterns: matches = re.findall(pattern, source) for m in matches: m = m.strip().rstrip(':-') if len(m) > 5 and m not in certs: certs.append(m) return certs[:10] def extract_hobbies(text: str) -> list: """ Extract hobbies and interests from resume text. Returns: List of hobby strings. """ section = _find_section(text, ["hobbies", "interests", "activities", "extra-curricular", "extracurricular"]) if not section: return [] hobbies = [] for line in section.split('\n'): line = line.strip().lstrip('•–-*').strip() if 2 < len(line) < 60: # Split by commas too (e.g. "Reading, Music, Cricket") parts = [p.strip() for p in line.split(',')] for p in parts: if p and p not in hobbies: hobbies.append(p.title()) return hobbies[:10] # ============================================================================= # 6. RECENCY FLAG # ============================================================================= def extract_recency(text: str, skills: list) -> str: """ Determine how recently the candidate used their key skills. Logic: - Find the most recent employment date in the resume - If last job ended within RECENCY_STALE_YEARS → "Recent" - If last job ended more than RECENCY_STALE_YEARS ago → "Stale" - If no dates found → "Unknown" Args: text: Raw resume text skills: Extracted skills list (unused for now, reserved for future) Returns: "Recent" | "Stale" | "Unknown" """ now = datetime.now() most_recent = None # Look for "Present" / "Current" — candidate is currently employed if re.search(r'\b(?:present|current|now)\b', text, re.IGNORECASE): return "Recent" # Find all 4-digit years mentioned in the resume years = re.findall(r'\b(20\d{2})\b', text) if years: latest_year = max(int(y) for y in years) years_ago = now.year - latest_year if years_ago <= config.RECENCY_STALE_YEARS: return "Recent" else: return "Stale" return "Unknown" # ============================================================================= # INTERNAL HELPER — Section Finder # ============================================================================= def _find_section(text: str, keywords: list) -> str: """ Find a named section in a resume and return its content. Looks for lines that match section header keywords, then returns everything until the next section header (all-caps or title-case line). Args: text: Full resume text keywords: List of keywords to identify the section Returns: Section content string, or empty string if not found. """ lines = text.split('\n') section_lines = [] in_section = False # Section header pattern — all caps or title case, short line header_pattern = re.compile(r'^[A-Z][A-Z\s&/]{2,30}$') for i, line in enumerate(lines): stripped = line.strip() # Detect section start if any(kw.lower() in stripped.lower() for kw in keywords): in_section = True continue if in_section: # Stop at next section header if (header_pattern.match(stripped) and len(stripped) > 3 and stripped not in ['AND', 'OR', 'THE']): break section_lines.append(line) return '\n'.join(section_lines).strip() # ============================================================================= # LLM EXTRACTION — Primary method for field labelling # ============================================================================= # Uses Groq (free) or OpenAI to intelligently label every field from raw text. # Falls back to regex-based extraction when no API key is available. # LLM extraction is significantly more reliable because it understands context — # it can tell "Python" in "fluent in Python and Java" is a skill, and # "Python" in "born in Python, Mississippi" is not. _LLM_EXTRACTOR_OK = False _llm_extractor_client = None _llm_extractor_model = "" def user_config_dir() -> str: """ OS-standard per-user config directory for keys. Windows: %LOCALAPPDATA%\\ResumeAnalyzer macOS : ~/Library/Application Support/ResumeAnalyzer Linux : $XDG_CONFIG_HOME/resume_analyzer (or ~/.config/resume_analyzer) """ if sys.platform == "win32": base = os.environ.get("LOCALAPPDATA") or os.path.expanduser("~") return os.path.join(base, "ResumeAnalyzer") if sys.platform == "darwin": return os.path.expanduser("~/Library/Application Support/ResumeAnalyzer") base = os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config") return os.path.join(base, "resume_analyzer") def user_env_path() -> str: return os.path.join(user_config_dir(), ".env") def _load_user_api_keys() -> None: """ Read API keys from the per-user .env file (if it exists) and inject them into os.environ — but ONLY for variables not already set. Environment variables always win, so devs running locally keep using their shell env. """ path = user_env_path() if not os.path.exists(path): return try: with open(path, encoding="utf-8") as f: for line in f: line = line.strip() if not line or line.startswith("#") or "=" not in line: continue key, _, val = line.partition("=") key = key.strip() val = val.strip().strip('"').strip("'") if key and val and key not in os.environ: os.environ[key] = val except Exception as exc: print(f"[Extractor] couldn't read {path}: {type(exc).__name__}: {exc}") def _init_llm_extractor(): """ Initialise LLM client for extraction. Priority order: 1. Anthropic Claude (best quality — install: pip install anthropic) 2. Groq via OpenAI-compatible API (openai package already installed) 3. OpenAI directly """ global _LLM_EXTRACTOR_OK, _llm_extractor_client, _llm_extractor_model # User-config fallback: when the app is installed via the Windows installer # there's no project-level .env. The Settings UI writes per-user keys to # the OS's standard user-config directory; honour those if the environment # doesn't already define them. _load_user_api_keys() # ── Option 1: Anthropic Claude (best at understanding context) ──────────── anthropic_key = os.environ.get("ANTHROPIC_API_KEY", "") if anthropic_key: try: import anthropic as _anthropic _llm_extractor_client = _anthropic.Anthropic(api_key=anthropic_key) _llm_extractor_model = "claude-haiku-4-5-20251001" _LLM_EXTRACTOR_OK = True print("[Extractor] LLM: Anthropic Claude (claude-haiku-4-5)") return except ImportError: print("[Extractor] anthropic package not installed — pip install anthropic") # ── Option 2: Groq via OpenAI-compatible API (no groq package needed) ──── groq_key = os.environ.get("GROQ_API_KEY", "") if groq_key: try: from openai import OpenAI _llm_extractor_client = OpenAI( api_key = groq_key, base_url = "https://api.groq.com/openai/v1", ) _llm_extractor_model = "llama-3.3-70b-versatile" _LLM_EXTRACTOR_OK = True print("[Extractor] LLM: Groq llama-3.3-70b-versatile (OpenAI-compat)") return except ImportError: print("[Extractor] openai package not installed") # ── Option 3: OpenAI directly ───────────────────────────────────────────── openai_key = os.environ.get("OPENAI_API_KEY", "") if openai_key: try: from openai import OpenAI _llm_extractor_client = OpenAI(api_key=openai_key) _llm_extractor_model = "gpt-4o-mini" _LLM_EXTRACTOR_OK = True print("[Extractor] LLM: OpenAI gpt-4o-mini") return except ImportError: pass print("[Extractor] WARNING: No LLM API key found — falling back to regex extraction only.") print(" Set GROQ_API_KEY in .env for free LLM-powered extraction.") _init_llm_extractor() _LLM_EXTRACTION_PROMPT = """Extract structured information from this resume text. Return ONLY valid JSON. No explanation, no markdown fences. Extract these fields: - candidate_name: The person's full name ONLY (given name + surname). Must be a real person's name found in the first 3 lines. NOT a job title (e.g. NOT "Financial Controller.txt"), NOT a location (e.g. NOT "Delhi NCR"), NOT a company name. (string) - email: Email address (string or "") - phone: Phone number digits only (string or "") - age: Age in years if mentioned, else 0 (number) - location: City and state/country if mentioned (string or "") - linkedin_url: LinkedIn profile URL if present (string or "") - github_url: GitHub profile URL if present, e.g. github.com/username (string or "") - portfolio_url: Personal portfolio / personal website URL if present (must NOT be linkedin.com or github.com — those go in their own fields) (string or "") - resume_summary: Write a 2-3 sentence professional summary of this candidate based on their overall profile. Include their years of experience, domain, strongest skills, and any standout achievement. (string) - technical_skills: List of technical/hard skills only (list of strings) - soft_skills: List of interpersonal/behavioural skills (list of strings) - languages: List of spoken/written languages the candidate knows (list of strings) - total_experience_years: Total years of full-time work experience, NOT counting internships (number) - job_titles: All full-time job titles held, most recent first (list of strings) - companies: List of REAL employer organisation names ONLY — i.e. the legally-registered entity the candidate was paid by. Each entry MUST look like a company brand. ✓ GOOD: "Infosys", "HDFC Bank", "Groz Engineering Tools Pvt. Ltd.", "Tata Consultancy Services", "Indian Armour Systems Pvt. Ltd." ✗ BAD — NEVER include any of these as a "company": - Cities/regions: "Delhi-NCR", "Mumbai", "Bangalore", "Gurgaon" - Job functions/departments: "Operations", "Insurance Operations", "Credit Management", "Business Analysis", "Operations Management", "Treasury & Cash Management" - Section headers from the resume: "EXPERIENCE", "PROFESSIONAL EXPERIENCE", "Work History" - Bullet-point fragments: "• Executed daily reviews", "Handled client queries", "Led a team of 5" - Acronyms-only: "KPI", "API", "PROGRAM &" (list of strings) - last_company: The MOST RECENT employer's organisation name ONLY. MUST be a real registered company (e.g. "HDFC Bank", "Infosys", "Groz Engineering Tools Pvt. Ltd."). MUST NOT be a city, department, job function, or section header. If you cannot identify a clear company name, return "". (string or "") - company_tenures: For every full-time role (NOT internships, NOT projects), one entry formatted EXACTLY as: "Company Name | Role Title | Start – End". Use full month names + 4-digit years when the resume gives them (e.g. "May 2021 – March 2023"); use year-only when months are not stated (e.g. "2018 – 2021"). Use the literal word "Present" for current/ongoing roles — do NOT compute or substitute the current year, the application will handle that. Example outputs: "Groz Engineering Tools Pvt. Ltd. | Production Engineer | Oct 2014 – Feb 2017", "HDFC Bank | Branch Manager | 2018 – 2021", "TCS | Senior Data Analyst | Jun 2022 – Present". Most recent role first. Company name MUST NOT include the city. (list of strings) - avg_tenure_months: Average months per full-time role (number or 0) - internships: List of internship descriptions — each as "Company | Role | Duration" (list of strings) - internship_count: Number of internships (number) - education_degree: Highest degree e.g. B.Tech, MBA, M.Tech (string or "") - education_field: Field of study e.g. Computer Science (string or "") - education_institution: University or college name (string or "") - graduation_year: Year of most recent graduation (number or 0) - certifications: List of certification names (list of strings) - projects: TECHNICAL PROJECTS the candidate personally built/designed/researched. A project is a tangible deliverable like an app, model, dashboard, system, or research study. STRICT RULES: (a) NEVER list employers/companies here — those belong in `companies`. (b) NEVER list job descriptions or work-history bullets here. (c) Each entry must look like a project title with an optional description, e.g. "Sales Revenue Dashboard — Automated monthly reporting using Python" or "Customer Churn Prediction Model — Reduced churn by 12%". If the resume has no clearly-labelled Projects section, return an empty list []. (list of strings) - achievements: List of awards, recognitions, hackathon wins, publications (list of strings) - hobbies: List of hobbies and interests (list of strings) - recency_flag: "Recent" if currently employed or worked within 2 years, "Stale" if last job was more than 2 years ago, "Unknown" if unclear - pros: Top 3 strengths of this candidate as a recruiter would see them (list of 3 strings) - cons: Top 3 weaknesses or gaps in this resume (list of 3 strings) Resume text: {resume_text} Return ONLY the JSON object. Every field must be present — use "" or [] or 0 for missing values.""" def _extract_via_llm(raw_text: str) -> dict: """ Use LLM to extract all structured fields from resume text. Returns a flat dict with all fields, or empty dict on failure. The caller merges this with regex-based fallbacks for any missing fields. """ if not _LLM_EXTRACTOR_OK or not _llm_extractor_client: return {} # Send the FULL resume text — every modern LLM we use (Claude haiku-4.5 # ≥200k, Groq llama-3.3-70b ≥128k, gpt-4o-mini ≥128k) has plenty of headroom # for even a 10-page CV. The previous 6 000-char truncation was the main # reason late-page employers (page 3+) were silently dropped from # company_tenures, which broke total-experience accuracy. # Cap at 60 000 chars (~15 k tokens) as a safety belt for pathological CVs. text_snippet = raw_text[:60000].strip() # Retry transient errors (rate limits, brief network blips). For # per-day token-cap rate limits, falling back to a smaller Groq model # (8B instant) keeps the pipeline producing usable extractions instead # of silently giving up on the LLM entirely. last_exc = None raw = "" fallback_models = ( [_llm_extractor_model, "llama-3.1-8b-instant"] if "llama-3.3-70b" in _llm_extractor_model else [_llm_extractor_model] ) for model_name in fallback_models: # The 8B fallback has only ~8k token context — truncate the resume # text much harder when we're using it, otherwise we get 413s. if "8b" in model_name.lower(): snippet_for_model = text_snippet[:12000] else: snippet_for_model = text_snippet prompt = _LLM_EXTRACTION_PROMPT.format(resume_text=snippet_for_model) for attempt in range(3): try: # Anthropic client has a different API surface if hasattr(_llm_extractor_client, "messages"): response = _llm_extractor_client.messages.create( model = model_name, max_tokens = 3000, messages = [{"role": "user", "content": prompt}], ) raw = response.content[0].text.strip() else: response = _llm_extractor_client.chat.completions.create( model = model_name, messages = [{"role": "user", "content": prompt}], max_tokens = 3000, temperature = 0.1, ) raw = response.choices[0].message.content.strip() break # success except Exception as exc: msg = str(exc) last_exc = exc # On daily-quota rate-limits, don't waste retries on the same # model — break out to the fallback model immediately. if "tokens per day" in msg or "TPD" in msg: break # Per-minute rate-limit: short backoff and retry wait = 2 ** attempt m = re.search(r"try again in\s+([\d.]+)s", msg, re.I) if m: wait = min(30, float(m.group(1)) + 1.0) time.sleep(wait) if raw: break # found a working model if not raw: raise (last_exc or RuntimeError("LLM call failed")) try: # Strip accidental markdown fences raw = re.sub(r"^```(?:json)?|```$", "", raw, flags=re.MULTILINE).strip() parsed = json.loads(raw) if not isinstance(parsed, dict): return {} # Normalise list fields list_fields = [ "technical_skills", "soft_skills", "languages", "job_titles", "companies", "certifications", "projects", "hobbies", "pros", "cons", "internships", "achievements", "company_tenures", ] for field in list_fields: val = parsed.get(field, []) if not isinstance(val, list): parsed[field] = [str(val)] if val else [] return parsed except json.JSONDecodeError as e: print(f"[Extractor] LLM returned invalid JSON: {e}") return {} except Exception as e: print(f"[Extractor] LLM extraction failed: {type(e).__name__}: {e}") return {} # ============================================================================= # MASTER FUNCTION — Call This From main.py # ============================================================================= def build_resume_record(file_name: str, raw_text: str) -> dict: """ Build a complete, DB-ready resume record from raw text. EXTRACTION STRATEGY — two-pass, best-of-both: Pass 1 — LLM extraction (Groq or OpenAI): Sends raw text to an LLM with a structured prompt. Returns a labelled JSON dict for all fields. Reliable because LLMs understand context — they can distinguish "Python (skill)" from "Python (place name)" and correctly identify which lines are education vs experience vs projects. Pass 2 — Regex/rule-based fallback: Runs for any field that LLM left empty or returned invalid data for. Uses the existing regex extractors as safety net. Always runs for: skills taxonomy matching (expands LLM skill list), contact info (email/phone regex is very reliable). The final record merges both passes — LLM fields take priority, regex fills gaps. Args: file_name : Original filename e.g. "priya_sharma.pdf" raw_text : Cleaned text from parser.extract_text() Returns: Dict matching resumes table schema, ready for DB insert. All list fields are JSON strings. """ # ── Pass 1: LLM extraction ──────────────────────────────────────────────── llm = _extract_via_llm(raw_text) # ── Pass 2: Regex fallbacks ─────────────────────────────────────────────── name_regex = extract_candidate_name(raw_text, file_name) contact = extract_contact(raw_text) url_bundle = extract_urls(raw_text) skills_regex = extract_skills(raw_text) # taxonomy-matched skills soft_regex = extract_soft_skills(raw_text) exp_regex = extract_experience(raw_text) edu_regex = extract_education(raw_text) projects_regex = extract_projects(raw_text) certs_regex = extract_certifications(raw_text) hobbies_regex = extract_hobbies(raw_text) recency_regex = extract_recency(raw_text, skills_regex) # ── Merge: LLM takes priority, regex fills gaps ─────────────────────────── # Identity — name needs careful handling: # If the filename gives a clean ≥2-word name (e.g. "Gaurav Joshi" from # "Gaurav_Joshi_Resume pdf.pdf"), that is more reliable than an LLM # extraction that may read garbled OCR text (e.g. "AURAV OSHI"). # Rule: prefer filename when it gives ≥2 words AND LLM gives ≤ same words. # Only let LLM override when it gives strictly MORE words (i.e. a more complete name). _llm_name = (llm.get("candidate_name") or "").strip() _filename_name = _name_from_filename(file_name) _llm_words = len(_llm_name.split()) if _llm_name != "" else 0 _file_words = len(_filename_name.split()) if _filename_name != "Unknown" else 0 if _file_words >= 2 and _llm_words <= _file_words: # Filename gives a solid 2+ word name and LLM gives same or fewer — # trust the filename (immune to OCR errors in PDF text) name = _filename_name elif _llm_name: # LLM gives more words (more complete) or filename only has 1 word name = _llm_name else: name = name_regex # spaCy / line-scan fallback age = int(llm.get("age") or 0) location = (llm.get("location") or "").strip() # If LLM didn't find a location, fall back to Indian-city database match if not location: location = _extract_location_from_text(raw_text) # URLs — LLM gives only linkedin_url; the regex bundle adds github, # portfolio and any other clickable URLs found anywhere in the CV. linkedin = (llm.get("linkedin_url") or "").strip() or url_bundle["linkedin_url"] github_url = (llm.get("github_url") or "").strip() or url_bundle["github_url"] portfolio_url = (llm.get("portfolio_url")or "").strip() or url_bundle["portfolio_url"] other_urls = url_bundle["other_urls"] # Summary — LLM only (needs full context understanding) resume_summary = (llm.get("resume_summary") or "").strip() # Skills — three-layer routing: # 1. LLM overrides (data/skill_softness_overrides.csv) are authoritative # for any label they cover. These are recruiter-context labels # assigned by llama-3.3-70b and (optionally) human-reviewed. # 2. Rule-based fallback (regex denylist + ESCO transversal lookup) for # labels not in the overrides. # 3. Subtractive merge below guarantees no skill appears in both lists. # The legacy JobBERT softness classifier is loaded but only consulted # when config.USE_SOFTNESS_CLASSIFIER = True (off by default; the LLM # overrides are usually better). llm_skills = [s.strip() for s in llm.get("technical_skills", []) if s.strip()] llm_soft = [s.strip() for s in llm.get("soft_skills", []) if s.strip()] rule_soft = list({s.lower(): s for s in soft_regex + llm_soft}.values()) rule_hard = list({s.lower(): s for s in skills_regex + llm_skills}.values()) # Apply LLM overrides: pull anything marked soft out of rule_hard, push # anything marked hard out of rule_soft. "language" and "other" labels # drop out of the skill lists entirely (they belong in `languages` / # nowhere). soft_final: list[str] = [] hard_final: list[str] = [] for s in rule_hard: ov = _SKILL_OVERRIDES.get(s.lower()) if ov: if ov["category"] == "soft": soft_final.append(s) elif ov["category"] == "hard": hard_final.append(s) # "language" / "other" → discard else: hard_final.append(s) for s in rule_soft: ov = _SKILL_OVERRIDES.get(s.lower()) if ov: if ov["category"] == "soft": soft_final.append(s) elif ov["category"] == "hard": hard_final.append(s) else: soft_final.append(s) # Optional in-flight classifier promotion for OOV labels. soft_keys = {s.lower() for s in soft_final} if _SOFTNESS_CLF is not None and getattr(config, "USE_SOFTNESS_CLASSIFIER", False): uncovered = [s for s in hard_final if s.lower() not in _SKILL_OVERRIDES and s.lower() not in soft_keys] if uncovered: for label, p in _classify_softness(uncovered).items(): if p >= _SOFTNESS_THRESHOLD: soft_final.append(label) soft_keys.add(label.lower()) merged_soft = list({s.lower(): s for s in soft_final}.values()) merged_skills = list({s.lower(): s for s in hard_final if s.lower() not in soft_keys}.values()) # Languages languages = [s.strip() for s in llm.get("languages", []) if s.strip()] # Full-time experience. # The regex scanner runs across the FULL resume text and now classifies # every date range as "work" / "internship" / "non_work" (education, # project, award …) using the surrounding context. That gives us a clean # split between full-time experience and internship time, computed # against datetime.now() (pipeline-run date) rather than the stale # "X years experience" string the candidate typed at CV-write time. regex_total_years = float(exp_regex["total_years"] or 0) regex_intern_yrs = float(exp_regex["internship_years"] or 0) total_years = (regex_total_years if regex_total_years > 0 else float(llm.get("total_experience_years") or 0)) internship_years = regex_intern_yrs # always trust regex split; LLM has no # comparable computed field job_titles = llm.get("job_titles", []) or exp_regex["job_titles"] companies = llm.get("companies", []) or exp_regex["companies"] # Scrub LLM hallucinations: remove location-only entries and trailing cities companies = [c for c in (_clean_company_name(c) for c in companies) if c] # last_company: try LLM first, then spaCy regex, then first cleaned company _lc_raw = (llm.get("last_company") or "").strip() if not _lc_raw: _lc_raw = exp_regex.get("last_company", "") last_company = _clean_company_name(_lc_raw) # If cleaning rejected it (location/function/garbage), fall back to the # first valid entry in the cleaned companies list. if not last_company: # Walk the companies list in order (most-recent-first from LLM) and # use the first one that is different from the rejected last_company for co in companies: if co and co != _lc_raw: last_company = co break avg_tenure = float(llm.get("avg_tenure_months") or exp_regex["avg_tenure_months"] or 0) # Internships (separate from full-time experience) internships = [s.strip() for s in llm.get("internships", []) if s.strip()] internship_count = int(llm.get("internship_count") or len(internships)) # Per-company tenure list — "Company | Role | Start – End" company_tenures = [s.strip() for s in llm.get("company_tenures", []) if s.strip()] # Regex fallback: LLM often skips this field (token budget / truncation); # build tenure entries from the raw text when the LLM returns nothing. if not company_tenures and companies: company_tenures = _build_tenures_from_text(raw_text, companies) # Anchor experience years to the pipeline-run date (NOT the CV write date). # We have two independent sources of truth: # (a) regex_total_years — full-CV date-range scan with internship-context # exclusion (set above). # (b) tenure_years — derived from the LLM's structured # company_tenures list, which can pick up roles the regex missed # because the start/end dates were on a different line from the # company name. # Take the MAX of the two so we never under-count when one source has a # blind spot. The exception is when regex strictly excludes a range as # internship — in that case we still want to trust the regex split. tenure_years = _years_from_tenures(company_tenures) if tenure_years > total_years: total_years = tenure_years # Education edu_degree = (llm.get("education_degree") or "").strip() or edu_regex.get("degree", "") edu_field = (llm.get("education_field") or "").strip() or edu_regex.get("field", "") edu_inst = (llm.get("education_institution") or "").strip() or edu_regex.get("institution", "") graduation_year = int(llm.get("graduation_year") or 0) education = { "degree" : edu_degree, "field" : edu_field, "institution" : edu_inst, "graduation_year": graduation_year, } # Projects, certifications, achievements, hobbies. # Trust the LLM's empty list when it ran — the prompt is explicit that an # absent "Projects" section in the CV must return []. Falling back to the # heuristic regex would invent project entries from random work bullets, # which is exactly what we want to avoid. if _LLM_EXTRACTOR_OK and "projects" in llm: projects = llm.get("projects") or [] else: projects = projects_regex # Scrub: drop any "project" that's actually a company-name/work-history leak projects = _filter_project_entries(projects, companies) certs = llm.get("certifications",[]) or certs_regex achievements = [s.strip() for s in llm.get("achievements", []) if s.strip()] hobbies = llm.get("hobbies", []) or hobbies_regex # Recency recency = (llm.get("recency_flag") or "").strip().title() if recency not in ("Recent", "Stale", "Unknown"): recency = recency_regex pros = [s.strip() for s in llm.get("pros", []) if s.strip()] cons = [s.strip() for s in llm.get("cons", []) if s.strip()] # Stability flag stability_flag = (avg_tenure < config.MIN_TENURE_MONTHS if avg_tenure > 0 else False) # ─── Rule-based fallback when LLM is unavailable (rate-limit / error) ────── # Each list is handled independently so that a strong candidate (no cons) # and a weak candidate (no pros) are both covered correctly. skill_count = len(merged_skills) if not pros: _pros = [] # Experience depth if total_years >= 5: _pros.append(f"{total_years:.0f}+ years of professional experience") elif total_years >= 2: _pros.append(f"{total_years:.0f} years of relevant experience") elif total_years >= 1: _pros.append(f"{total_years:.0f} year of work experience") # Skills breadth if skill_count >= 10: _pros.append(f"Strong technical breadth — {skill_count} skills detected") elif skill_count >= 5: _pros.append(f"{skill_count} technical skills identified") elif skill_count >= 1: _pros.append(f"{skill_count} technical skill(s) listed") # Recency if recency == "Recent": _pros.append("Currently employed / recently active") # Stable career if avg_tenure >= 18: _pros.append(f"Stable job history (avg {avg_tenure:.0f} months per role)") # Certifications if len(certs) >= 2: _pros.append(f"{len(certs)} certifications on record") elif len(certs) == 1: _pros.append("Holds a professional certification") # Education if edu_degree: _pros.append("Holds a " + edu_degree + (f" in {edu_field}" if edu_field else "")) # Catch-all — always ensure at least one entry if not _pros: _pros.append("Resume received — limited extractable data; manual review recommended") pros = _pros if not cons: _cons = [] # Experience gap if total_years == 0: _cons.append("No quantifiable work experience detected in resume") elif total_years < 2: _cons.append(f"Limited work experience ({total_years:.1f} yrs) — under typical threshold") # Skills gap if skill_count < 3: _cons.append("Very few technical skills detected — resume may need more detail") # Job hopping if stability_flag: _cons.append(f"Frequent job changes detected (avg {avg_tenure:.0f} months per role)") # Stale / inactive if recency == "Stale": _cons.append("Employment gap or outdated recent role detected") # No certifications for junior profiles if not certs and total_years < 3: _cons.append("No professional certifications listed") # Catch-all — strong candidates genuinely have no concerns if not _cons: _cons.append("No significant concerns identified from extracted resume data") cons = _cons # ─── Rule-based summary fallback ───────────────────────────────────────── if not resume_summary: _parts = [] if name and name != "Unknown": _parts.append(f"{name} is a") else: _parts.append("Candidate is a") if total_years >= 1: _parts.append(f"{total_years:.0f}-year professional") else: _parts.append("professional") if edu_degree: _parts.append(f"holding a {edu_degree}" + (f" in {edu_field}" if edu_field else "")) if merged_skills: top_skills = merged_skills[:4] _parts.append(f"with skills in {', '.join(top_skills)}") if last_company: _parts.append(f"(most recently at {last_company})") resume_summary = " ".join(_parts).strip().rstrip(",") + "." # Capitalise first letter if resume_summary: resume_summary = resume_summary[0].upper() + resume_summary[1:] return { # Identity "file_name" : file_name, "candidate_name" : name, "email" : contact["email"], "phone" : contact["phone"], "age" : age, "location" : location, "linkedin_url" : linkedin, "github_url" : github_url, "portfolio_url" : portfolio_url, "other_urls" : json.dumps(other_urls), # Summary "resume_summary" : resume_summary, # Raw text (kept for semantic matching) "resume_text" : raw_text, # Skills "skills" : json.dumps(merged_skills), "soft_skills" : json.dumps(merged_soft), "languages" : json.dumps(languages), # Full-time experience "experience_years" : total_years, "last_company" : last_company, "job_titles" : json.dumps(job_titles), "companies" : json.dumps(companies), "avg_tenure_months" : avg_tenure, # Internships — kept separately so they never inflate the work-exp total "internships" : json.dumps(internships), "internship_count" : internship_count, "internship_years" : internship_years, # Per-company tenure: ["Company | Role | 18 months", …] "company_tenures" : json.dumps(company_tenures), # Education "education" : json.dumps(education), "graduation_year" : graduation_year, # Other "certifications" : json.dumps(certs), "projects" : json.dumps(projects), "achievements" : json.dumps(achievements), "hobbies" : json.dumps(hobbies), # Scoring inputs (used by matcher.py) "recency_flag" : recency, "stability_flag" : stability_flag, # Batch "upload_batch" : datetime.now().strftime("%Y%m%d_%H%M%S"), "technical_skills": ", ".join(merged_skills), # was "skills" (JSON) "total_experience_years": total_years, # was "experience_years" "education_degree": edu_degree, "education_field": edu_field, "education_institution": edu_inst, "pros" : json.dumps(pros), "cons" : json.dumps(cons), } # ============================================================================= # SELF-TEST # Run directly: python engine/extractor.py # ============================================================================= if __name__ == "__main__": GREEN = "\033[92m" RED = "\033[91m" YELLOW = "\033[93m" BOLD = "\033[1m" RESET = "\033[0m" print(f"\n{BOLD}{'='*60}{RESET}") print(f"{BOLD} EXTRACTOR SELF-TEST{RESET}") print(f"{BOLD}{'='*60}{RESET}\n") # Check spaCy if _SPACY_OK: print(f" {GREEN}✓{RESET} spaCy loaded: en_core_web_sm") else: print(f" {YELLOW}⚠{RESET} spaCy not available — " f"run: python -m spacy download en_core_web_sm") print(f" {GREEN}✓{RESET} Skills taxonomy: {len(_SKILLS_TAXONOMY)} skills loaded\n") # Sample resume text SAMPLE = """ Priya Sharma priya.sharma@email.com | +91 9876543210 Mumbai, India EXPERIENCE Senior Data Analyst — Infosys, Mumbai Jan 2022 – Present - Built sales dashboards using Python, SQL, and Tableau - Automated reporting pipeline with Pandas and NumPy Data Analyst — Wipro, Pune June 2020 – Dec 2021 - Developed churn prediction model using scikit-learn - Created Power BI dashboards for leadership team EDUCATION B.Tech in Computer Science — VIT University, 2020 SKILLS Python, SQL, Tableau, Power BI, Machine Learning, Pandas, NumPy, scikit-learn, Git, Agile PROJECTS Sales Revenue Dashboard — Automated monthly reporting using Python Customer Churn Model — Reduced churn by 12% using ML classification CERTIFICATIONS AWS Cloud Practitioner, Google Data Analytics Certificate HOBBIES Reading, Hackathons, Cricket """ record = build_resume_record("priya_sharma.pdf", SAMPLE) passed = 0 failed = 0 def check(condition, label, got=""): global passed, failed if condition: print(f" {GREEN}✓{RESET} {label}" + (f" → {got}" if got else "")) passed += 1 else: print(f" {RED}✗{RESET} {label}" + (f" → got: {got}" if got else "")) failed += 1 check(record["candidate_name"] != "Unknown", "Name extracted", record["candidate_name"]) check("sharma" in record["email"].lower() or "@" in record["email"], "Email extracted", record["email"]) check(len(record["phone"]) >= 8, "Phone extracted", record["phone"]) skills = json.loads(record["skills"]) check(len(skills) >= 3, f"Skills extracted ({len(skills)} found)", ", ".join(skills[:5])) soft = json.loads(record["soft_skills"]) check(isinstance(soft, list), f"Soft skills extracted ({len(soft)} found)", ", ".join(soft[:3]) if soft else "none") check(record["experience_years"] > 0, f"Experience years", f"{record['experience_years']} yrs") check(record["avg_tenure_months"] > 0, f"Avg tenure", f"{record['avg_tenure_months']} months") check(record["recency_flag"] in ("Recent", "Stale", "Unknown"), f"Recency flag", record["recency_flag"]) edu = json.loads(record["education"]) check(edu.get("degree", "") != "", "Education extracted", f"{edu.get('degree')} in {edu.get('field')}") projects = json.loads(record["projects"]) check(len(projects) >= 1, f"Projects extracted ({len(projects)} found)", projects[0] if projects else "none") certs = json.loads(record["certifications"]) check(len(certs) >= 1, f"Certifications extracted ({len(certs)} found)", certs[0] if certs else "none") # All required keys present required_keys = [ "file_name", "candidate_name", "email", "phone", "resume_text", "skills", "soft_skills", "experience_years", "last_company", "job_titles", "education", "certifications", "projects", "hobbies", "avg_tenure_months", "recency_flag", "upload_batch" ] missing_keys = [k for k in required_keys if k not in record] check(len(missing_keys) == 0, "All DB schema keys present", f"Missing: {missing_keys}" if missing_keys else "") print(f"\n Passed: {GREEN}{passed}{RESET} | " f"Failed: {RED}{failed}{RESET}") if failed == 0: print(f"\n {GREEN}{BOLD}✓ All tests passed — extractor.py is ready.{RESET}") print(f"\n Next step: tell Claude 'ready for Step 5 (jd_parser)'\n") else: print(f"\n {YELLOW}Some tests failed — check output above.{RESET}") print(f" Partial failures are okay if spaCy isn't installed yet.\n") print("="*60 + "\n")