AnishDev's picture
initial code and logic pushed
ff7b177
Raw History Blame Contribute Delete
12 kB
"""
Extract structured requirements from any JD text (PDF-extracted).
Works generically β€” not hardcoded to AI Engineer / Backend Engineer.
Extracts:
- Role title
- Engineer type (inferred from title + content)
- Minimum years of experience
- Required skills (from Requirements / Core Requirements section)
- Preferred skills (from Preferred Qualifications section)
- Key responsibilities
"""
import re
import uuid
from typing import Optional
from app.models.domain import JDRequirements, SkillRequirement
from app.pipeline.skill_parser import (
parse_skill_token,
normalize_skill_name,
SKILL_ALIASES,
)
# ─── Engineer type inference ──────────────────────────────────────────────────
_TYPE_PATTERNS: list[tuple[list[str], str]] = [
(["ai engineer", "ml engineer", "machine learning engineer", "nlp engineer",
"data scientist", "applied scientist"], "AI Engineer"),
(["backend engineer", "back-end engineer", "back end engineer",
"server side engineer", "api engineer", "node.js developer",
"python developer", "java developer"], "Backend Engineer"),
(["frontend engineer", "front-end engineer", "front end engineer",
"ui engineer", "react developer", "angular developer",
"vue developer", "web developer"], "Frontend Engineer"),
(["fullstack", "full stack", "full-stack"], "Full Stack Engineer"),
(["data engineer", "etl developer", "pipeline engineer"], "Data Engineer"),
(["devops engineer", "sre", "platform engineer", "cloud engineer",
"infrastructure engineer"], "DevOps Engineer"),
(["mobile engineer", "ios developer", "android developer",
"react native developer", "flutter developer"], "Mobile Engineer"),
(["software engineer", "software developer", "sde", "swe"], "Software Engineer"),
]
def infer_engineer_type(title: str, text: str) -> Optional[str]:
combined = (title + " " + text[:500]).lower()
for keywords, eng_type in _TYPE_PATTERNS:
if any(kw in combined for kw in keywords):
return eng_type
return None
# ─── Years extraction ─────────────────────────────────────────────────────────
_YEARS_PATTERNS = [
r"minimum\s+(\d+(?:\.\d+)?)\s*\+?\s*years?",
r"at\s+least\s+(\d+(?:\.\d+)?)\s*\+?\s*years?",
r"(\d+)\+?\s*years?\s+of\s+(?:relevant\s+)?experience",
r"(\d+)\+?\s*years?\s+(?:working\s+)?(?:in|with|of)",
r"experience[:\s]+(\d+)\+?\s*years?",
r"(\d+)\s*-\s*\d+\s*years?\s+of\s+experience",
]
def extract_min_years(text: str) -> float:
lower = text.lower()
for pattern in _YEARS_PATTERNS:
m = re.search(pattern, lower)
if m:
try:
return float(m.group(1))
except ValueError:
pass
return 0.0
# ─── Title extraction ─────────────────────────────────────────────────────────
def extract_title(text: str) -> str:
lines = [l.strip() for l in text.split("\n") if l.strip()]
# First non-empty line is usually the title
if lines:
# Take up to first 80 chars, strip markdown artifacts
candidate = re.sub(r"[#*_]", "", lines[0]).strip()
if 3 < len(candidate) < 120:
return candidate
return "Unknown Role"
# ─── Tech skills vocabulary ───────────────────────────────────────────────────
# Comprehensive vocabulary for matching skills mentioned in JD text
TECH_VOCAB: set[str] = {
# Languages
"python", "java", "javascript", "typescript", "go", "golang", "rust",
"c++", "c#", "ruby", "kotlin", "swift", "scala", "r", "php",
# Python frameworks
"fastapi", "django", "flask", "celery", "sqlalchemy",
# JS frameworks
"react", "angular", "vue", "nextjs", "express", "nodejs",
# AI/ML
"rag", "llm", "embeddings", "vector databases", "langchain", "langgraph",
"tensorflow", "pytorch", "scikit-learn", "huggingface", "transformers",
"generative ai", "nlp", "computer vision", "mlops", "fine-tuning",
"prompt engineering", "openai", "anthropic", "bedrock",
"pandas", "numpy", "matplotlib", "seaborn",
"apache spark", "airflow", "mlflow", "weights & biases",
# Databases
"postgresql", "mysql", "mongodb", "redis", "elasticsearch",
"cassandra", "dynamodb", "sqlite", "oracle", "mssql",
"qdrant", "pinecone", "weaviate", "chroma", "faiss",
# Cloud / DevOps
"aws", "gcp", "azure", "docker", "kubernetes", "terraform",
"github actions", "jenkins", "ci/cd", "lambda", "s3", "ec2",
# Messaging / async
"kafka", "rabbitmq", "celery", "sqs", "pubsub",
# APIs / patterns
"rest apis", "graphql", "grpc", "websockets", "microservices",
"serverless", "api gateway",
# Tools
"git", "jira", "confluence",
# Concepts
"system design", "distributed systems", "agile", "tdd",
"data structures", "algorithms",
# Java ecosystem
"springboot", "spring", "hibernate", "maven", "gradle",
# .NET
".net", "asp.net",
}
def extract_skills_from_text(
text: str, is_required: bool
) -> list[SkillRequirement]:
"""
Scan text for known tech skills and extract with proficiency context.
Returns deduplicated list.
"""
lower = text.lower()
found: dict[str, SkillRequirement] = {}
for skill in sorted(TECH_VOCAB, key=len, reverse=True):
if skill not in lower:
continue
# Find position and extract surrounding context (40 chars before)
idx = lower.find(skill)
context_before = lower[max(0, idx - 40): idx]
context_after = lower[idx + len(skill): idx + len(skill) + 30]
context = context_before + " " + context_after
# Determine level from context qualifiers
level = _level_from_context(context)
normalized = normalize_skill_name(skill)
if normalized not in found:
found[normalized] = SkillRequirement(
name=normalized,
level=level,
is_required=is_required,
raw=skill,
)
return list(found.values())
def _level_from_context(context: str) -> int:
"""Infer required proficiency from surrounding words."""
from app.pipeline.skill_parser import _QUALIFIER_MAP, _SORTED_QUALIFIERS
lower = context.lower()
for qualifier in _SORTED_QUALIFIERS:
if qualifier in lower:
return _QUALIFIER_MAP[qualifier]
return 3 # default: intermediate
# ─── Section extraction ───────────────────────────────────────────────────────
_REQUIRED_HEADERS = [
"core requirements", "requirements", "required qualifications",
"must have", "what you need", "you will need", "technical requirements",
"skills required",
]
_PREFERRED_HEADERS = [
"preferred qualifications", "preferred", "nice to have",
"bonus points", "brownie points", "good to have",
"what would be great", "plus",
]
_RESPONSIBILITY_HEADERS = [
"responsibilities", "what you'll do", "what you will do",
"your role", "duties", "key responsibilities",
"role and responsibilities",
]
def _extract_section(text: str, headers: list[str]) -> str:
"""Return text between the first matching header and the next section break."""
lower = text.lower()
start = -1
for header in headers:
idx = lower.find(header)
if idx != -1:
start = idx
break
if start == -1:
return ""
# Find next section break (another header or double newline after 50+ chars)
end = len(text)
for other_headers in [_REQUIRED_HEADERS, _PREFERRED_HEADERS, _RESPONSIBILITY_HEADERS]:
for h in other_headers:
idx = lower.find(h, start + 20)
if idx != -1 and idx < end:
end = idx
return text[start:end]
def _parse_bullets(section_text: str) -> list[str]:
"""Extract bullet points from a section."""
bullets: list[str] = []
for line in section_text.split("\n"):
line = line.strip()
# Remove bullet markers
line = re.sub(r"^[β€’Β·\-\*β—βž€β–Ίβ–Έβ–ͺ◦‣⁃]\s*", "", line).strip()
if len(line) > 5:
bullets.append(line)
return bullets
# ─── Skills from "Skills Required" comma list ─────────────────────────────────
def _parse_skills_section(text: str) -> list[str]:
"""Find 'Skills Required' or similar and return comma/newline split items."""
lower = text.lower()
for header in ["skills required", "technical skills", "technologies", "key skills"]:
idx = lower.find(header)
if idx != -1:
# Grab next 500 chars
snippet = text[idx: idx + 500]
# Split by comma or newline
tokens = re.split(r"[,\n]", snippet)
# Skip the header line itself
result = []
for t in tokens[1:]:
t = t.strip().strip("β€’Β·-*●").strip()
if 2 < len(t) < 60:
result.append(t)
return result
return []
# ─── Main JD parser ───────────────────────────────────────────────────────────
def parse_jd(text: str, jd_id: Optional[str] = None) -> JDRequirements:
if jd_id is None:
jd_id = str(uuid.uuid4())[:8]
title = extract_title(text)
engineer_type = infer_engineer_type(title, text)
min_years = extract_min_years(text)
# Extract sections
req_section = _extract_section(text, _REQUIRED_HEADERS)
pref_section = _extract_section(text, _PREFERRED_HEADERS)
resp_section = _extract_section(text, _RESPONSIBILITY_HEADERS)
# Parse skills from dedicated sections using vocab matching
required_skills = extract_skills_from_text(req_section or text[:800], is_required=True)
preferred_skills = extract_skills_from_text(pref_section, is_required=False)
# Also parse comma-separated "Skills Required" section if present.
# ONLY add tokens that match known TECH_VOCAB β€” prevents job titles ("AI Engineer"),
# section headers ("Overview"), and sentence fragments ("and managing iot") from
# leaking into the required skills list.
_normalized_vocab = {normalize_skill_name(s) for s in TECH_VOCAB}
comma_skills = _parse_skills_section(text)
existing_names = {s.name for s in required_skills}
for raw_skill in comma_skills:
entry = parse_skill_token(raw_skill)
if entry.name and entry.name not in existing_names and entry.name in _normalized_vocab:
required_skills.append(SkillRequirement(
name=entry.name,
level=entry.level,
is_required=True,
raw=raw_skill,
))
existing_names.add(entry.name)
# Responsibilities as plain bullets
responsibilities = _parse_bullets(resp_section)[:8]
# Fallback: if we extracted 0 required skills, scan full text
if not required_skills:
required_skills = extract_skills_from_text(text, is_required=True)
return JDRequirements(
id=jd_id,
title=title,
engineer_type=engineer_type,
min_years=min_years,
required_skills=required_skills,
preferred_skills=preferred_skills,
responsibilities=responsibilities,
raw_text=text[:200],
full_text=text,
)