""" Extract structured requirements from any JD text (PDF-extracted). Works generically — not hardcoded to AI Engineer / Backend Engineer. Extracts: - Role title - Engineer type (inferred from title + content) - Minimum years of experience - Required skills (from Requirements / Core Requirements section) - Preferred skills (from Preferred Qualifications section) - Key responsibilities """ import re import uuid from typing import Optional from app.models.domain import JDRequirements, SkillRequirement from app.pipeline.skill_parser import ( parse_skill_token, normalize_skill_name, SKILL_ALIASES, ) # ─── Engineer type inference ────────────────────────────────────────────────── _TYPE_PATTERNS: list[tuple[list[str], str]] = [ (["ai engineer", "ml engineer", "machine learning engineer", "nlp engineer", "data scientist", "applied scientist"], "AI Engineer"), (["backend engineer", "back-end engineer", "back end engineer", "server side engineer", "api engineer", "node.js developer", "python developer", "java developer"], "Backend Engineer"), (["frontend engineer", "front-end engineer", "front end engineer", "ui engineer", "react developer", "angular developer", "vue developer", "web developer"], "Frontend Engineer"), (["fullstack", "full stack", "full-stack"], "Full Stack Engineer"), (["data engineer", "etl developer", "pipeline engineer"], "Data Engineer"), (["devops engineer", "sre", "platform engineer", "cloud engineer", "infrastructure engineer"], "DevOps Engineer"), (["mobile engineer", "ios developer", "android developer", "react native developer", "flutter developer"], "Mobile Engineer"), (["software engineer", "software developer", "sde", "swe"], "Software Engineer"), ] def infer_engineer_type(title: str, text: str) -> Optional[str]: combined = (title + " " + text[:500]).lower() for keywords, eng_type in _TYPE_PATTERNS: if any(kw in combined for kw in keywords): return eng_type return None # ─── Years extraction ───────────────────────────────────────────────────────── _YEARS_PATTERNS = [ r"minimum\s+(\d+(?:\.\d+)?)\s*\+?\s*years?", r"at\s+least\s+(\d+(?:\.\d+)?)\s*\+?\s*years?", r"(\d+)\+?\s*years?\s+of\s+(?:relevant\s+)?experience", r"(\d+)\+?\s*years?\s+(?:working\s+)?(?:in|with|of)", r"experience[:\s]+(\d+)\+?\s*years?", r"(\d+)\s*-\s*\d+\s*years?\s+of\s+experience", ] def extract_min_years(text: str) -> float: lower = text.lower() for pattern in _YEARS_PATTERNS: m = re.search(pattern, lower) if m: try: return float(m.group(1)) except ValueError: pass return 0.0 # ─── Title extraction ───────────────────────────────────────────────────────── def extract_title(text: str) -> str: lines = [l.strip() for l in text.split("\n") if l.strip()] # First non-empty line is usually the title if lines: # Take up to first 80 chars, strip markdown artifacts candidate = re.sub(r"[#*_]", "", lines[0]).strip() if 3 < len(candidate) < 120: return candidate return "Unknown Role" # ─── Tech skills vocabulary ─────────────────────────────────────────────────── # Comprehensive vocabulary for matching skills mentioned in JD text TECH_VOCAB: set[str] = { # Languages "python", "java", "javascript", "typescript", "go", "golang", "rust", "c++", "c#", "ruby", "kotlin", "swift", "scala", "r", "php", # Python frameworks "fastapi", "django", "flask", "celery", "sqlalchemy", # JS frameworks "react", "angular", "vue", "nextjs", "express", "nodejs", # AI/ML "rag", "llm", "embeddings", "vector databases", "langchain", "langgraph", "tensorflow", "pytorch", "scikit-learn", "huggingface", "transformers", "generative ai", "nlp", "computer vision", "mlops", "fine-tuning", "prompt engineering", "openai", "anthropic", "bedrock", "pandas", "numpy", "matplotlib", "seaborn", "apache spark", "airflow", "mlflow", "weights & biases", # Databases "postgresql", "mysql", "mongodb", "redis", "elasticsearch", "cassandra", "dynamodb", "sqlite", "oracle", "mssql", "qdrant", "pinecone", "weaviate", "chroma", "faiss", # Cloud / DevOps "aws", "gcp", "azure", "docker", "kubernetes", "terraform", "github actions", "jenkins", "ci/cd", "lambda", "s3", "ec2", # Messaging / async "kafka", "rabbitmq", "celery", "sqs", "pubsub", # APIs / patterns "rest apis", "graphql", "grpc", "websockets", "microservices", "serverless", "api gateway", # Tools "git", "jira", "confluence", # Concepts "system design", "distributed systems", "agile", "tdd", "data structures", "algorithms", # Java ecosystem "springboot", "spring", "hibernate", "maven", "gradle", # .NET ".net", "asp.net", } def extract_skills_from_text( text: str, is_required: bool ) -> list[SkillRequirement]: """ Scan text for known tech skills and extract with proficiency context. Returns deduplicated list. """ lower = text.lower() found: dict[str, SkillRequirement] = {} for skill in sorted(TECH_VOCAB, key=len, reverse=True): if skill not in lower: continue # Find position and extract surrounding context (40 chars before) idx = lower.find(skill) context_before = lower[max(0, idx - 40): idx] context_after = lower[idx + len(skill): idx + len(skill) + 30] context = context_before + " " + context_after # Determine level from context qualifiers level = _level_from_context(context) normalized = normalize_skill_name(skill) if normalized not in found: found[normalized] = SkillRequirement( name=normalized, level=level, is_required=is_required, raw=skill, ) return list(found.values()) def _level_from_context(context: str) -> int: """Infer required proficiency from surrounding words.""" from app.pipeline.skill_parser import _QUALIFIER_MAP, _SORTED_QUALIFIERS lower = context.lower() for qualifier in _SORTED_QUALIFIERS: if qualifier in lower: return _QUALIFIER_MAP[qualifier] return 3 # default: intermediate # ─── Section extraction ─────────────────────────────────────────────────────── _REQUIRED_HEADERS = [ "core requirements", "requirements", "required qualifications", "must have", "what you need", "you will need", "technical requirements", "skills required", ] _PREFERRED_HEADERS = [ "preferred qualifications", "preferred", "nice to have", "bonus points", "brownie points", "good to have", "what would be great", "plus", ] _RESPONSIBILITY_HEADERS = [ "responsibilities", "what you'll do", "what you will do", "your role", "duties", "key responsibilities", "role and responsibilities", ] def _extract_section(text: str, headers: list[str]) -> str: """Return text between the first matching header and the next section break.""" lower = text.lower() start = -1 for header in headers: idx = lower.find(header) if idx != -1: start = idx break if start == -1: return "" # Find next section break (another header or double newline after 50+ chars) end = len(text) for other_headers in [_REQUIRED_HEADERS, _PREFERRED_HEADERS, _RESPONSIBILITY_HEADERS]: for h in other_headers: idx = lower.find(h, start + 20) if idx != -1 and idx < end: end = idx return text[start:end] def _parse_bullets(section_text: str) -> list[str]: """Extract bullet points from a section.""" bullets: list[str] = [] for line in section_text.split("\n"): line = line.strip() # Remove bullet markers line = re.sub(r"^[•·\-\*●➤►▸▪◦‣⁃]\s*", "", line).strip() if len(line) > 5: bullets.append(line) return bullets # ─── Skills from "Skills Required" comma list ───────────────────────────────── def _parse_skills_section(text: str) -> list[str]: """Find 'Skills Required' or similar and return comma/newline split items.""" lower = text.lower() for header in ["skills required", "technical skills", "technologies", "key skills"]: idx = lower.find(header) if idx != -1: # Grab next 500 chars snippet = text[idx: idx + 500] # Split by comma or newline tokens = re.split(r"[,\n]", snippet) # Skip the header line itself result = [] for t in tokens[1:]: t = t.strip().strip("•·-*●").strip() if 2 < len(t) < 60: result.append(t) return result return [] # ─── Main JD parser ─────────────────────────────────────────────────────────── def parse_jd(text: str, jd_id: Optional[str] = None) -> JDRequirements: if jd_id is None: jd_id = str(uuid.uuid4())[:8] title = extract_title(text) engineer_type = infer_engineer_type(title, text) min_years = extract_min_years(text) # Extract sections req_section = _extract_section(text, _REQUIRED_HEADERS) pref_section = _extract_section(text, _PREFERRED_HEADERS) resp_section = _extract_section(text, _RESPONSIBILITY_HEADERS) # Parse skills from dedicated sections using vocab matching required_skills = extract_skills_from_text(req_section or text[:800], is_required=True) preferred_skills = extract_skills_from_text(pref_section, is_required=False) # Also parse comma-separated "Skills Required" section if present. # ONLY add tokens that match known TECH_VOCAB — prevents job titles ("AI Engineer"), # section headers ("Overview"), and sentence fragments ("and managing iot") from # leaking into the required skills list. _normalized_vocab = {normalize_skill_name(s) for s in TECH_VOCAB} comma_skills = _parse_skills_section(text) existing_names = {s.name for s in required_skills} for raw_skill in comma_skills: entry = parse_skill_token(raw_skill) if entry.name and entry.name not in existing_names and entry.name in _normalized_vocab: required_skills.append(SkillRequirement( name=entry.name, level=entry.level, is_required=True, raw=raw_skill, )) existing_names.add(entry.name) # Responsibilities as plain bullets responsibilities = _parse_bullets(resp_section)[:8] # Fallback: if we extracted 0 required skills, scan full text if not required_skills: required_skills = extract_skills_from_text(text, is_required=True) return JDRequirements( id=jd_id, title=title, engineer_type=engineer_type, min_years=min_years, required_skills=required_skills, preferred_skills=preferred_skills, responsibilities=responsibilities, raw_text=text[:200], full_text=text, )