Spaces:
Sleeping
Sleeping
Download app/pipeline/jd_parser.py from AnishDev/coderound-task-backend: direct link, hf CLI and curl.
- Browser
- Download file 12 kB
-
https://huggingface.co/spaces/AnishDev/coderound-task-backend/resolve/main/app/pipeline/jd_parser.py
- Command line
-
hf download hf://spaces/AnishDev/coderound-task-backend/app/pipeline/jd_parser.py
-
curl -L -o jd_parser.py https://huggingface.co/spaces/AnishDev/coderound-task-backend/resolve/main/app/pipeline/jd_parser.py
12 kB
| """ | |
| Extract structured requirements from any JD text (PDF-extracted). | |
| Works generically β not hardcoded to AI Engineer / Backend Engineer. | |
| Extracts: | |
| - Role title | |
| - Engineer type (inferred from title + content) | |
| - Minimum years of experience | |
| - Required skills (from Requirements / Core Requirements section) | |
| - Preferred skills (from Preferred Qualifications section) | |
| - Key responsibilities | |
| """ | |
| import re | |
| import uuid | |
| from typing import Optional | |
| from app.models.domain import JDRequirements, SkillRequirement | |
| from app.pipeline.skill_parser import ( | |
| parse_skill_token, | |
| normalize_skill_name, | |
| SKILL_ALIASES, | |
| ) | |
| # βββ Engineer type inference ββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| _TYPE_PATTERNS: list[tuple[list[str], str]] = [ | |
| (["ai engineer", "ml engineer", "machine learning engineer", "nlp engineer", | |
| "data scientist", "applied scientist"], "AI Engineer"), | |
| (["backend engineer", "back-end engineer", "back end engineer", | |
| "server side engineer", "api engineer", "node.js developer", | |
| "python developer", "java developer"], "Backend Engineer"), | |
| (["frontend engineer", "front-end engineer", "front end engineer", | |
| "ui engineer", "react developer", "angular developer", | |
| "vue developer", "web developer"], "Frontend Engineer"), | |
| (["fullstack", "full stack", "full-stack"], "Full Stack Engineer"), | |
| (["data engineer", "etl developer", "pipeline engineer"], "Data Engineer"), | |
| (["devops engineer", "sre", "platform engineer", "cloud engineer", | |
| "infrastructure engineer"], "DevOps Engineer"), | |
| (["mobile engineer", "ios developer", "android developer", | |
| "react native developer", "flutter developer"], "Mobile Engineer"), | |
| (["software engineer", "software developer", "sde", "swe"], "Software Engineer"), | |
| ] | |
| def infer_engineer_type(title: str, text: str) -> Optional[str]: | |
| combined = (title + " " + text[:500]).lower() | |
| for keywords, eng_type in _TYPE_PATTERNS: | |
| if any(kw in combined for kw in keywords): | |
| return eng_type | |
| return None | |
| # βββ Years extraction βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| _YEARS_PATTERNS = [ | |
| r"minimum\s+(\d+(?:\.\d+)?)\s*\+?\s*years?", | |
| r"at\s+least\s+(\d+(?:\.\d+)?)\s*\+?\s*years?", | |
| r"(\d+)\+?\s*years?\s+of\s+(?:relevant\s+)?experience", | |
| r"(\d+)\+?\s*years?\s+(?:working\s+)?(?:in|with|of)", | |
| r"experience[:\s]+(\d+)\+?\s*years?", | |
| r"(\d+)\s*-\s*\d+\s*years?\s+of\s+experience", | |
| ] | |
| def extract_min_years(text: str) -> float: | |
| lower = text.lower() | |
| for pattern in _YEARS_PATTERNS: | |
| m = re.search(pattern, lower) | |
| if m: | |
| try: | |
| return float(m.group(1)) | |
| except ValueError: | |
| pass | |
| return 0.0 | |
| # βββ Title extraction βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def extract_title(text: str) -> str: | |
| lines = [l.strip() for l in text.split("\n") if l.strip()] | |
| # First non-empty line is usually the title | |
| if lines: | |
| # Take up to first 80 chars, strip markdown artifacts | |
| candidate = re.sub(r"[#*_]", "", lines[0]).strip() | |
| if 3 < len(candidate) < 120: | |
| return candidate | |
| return "Unknown Role" | |
| # βββ Tech skills vocabulary βββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Comprehensive vocabulary for matching skills mentioned in JD text | |
| TECH_VOCAB: set[str] = { | |
| # Languages | |
| "python", "java", "javascript", "typescript", "go", "golang", "rust", | |
| "c++", "c#", "ruby", "kotlin", "swift", "scala", "r", "php", | |
| # Python frameworks | |
| "fastapi", "django", "flask", "celery", "sqlalchemy", | |
| # JS frameworks | |
| "react", "angular", "vue", "nextjs", "express", "nodejs", | |
| # AI/ML | |
| "rag", "llm", "embeddings", "vector databases", "langchain", "langgraph", | |
| "tensorflow", "pytorch", "scikit-learn", "huggingface", "transformers", | |
| "generative ai", "nlp", "computer vision", "mlops", "fine-tuning", | |
| "prompt engineering", "openai", "anthropic", "bedrock", | |
| "pandas", "numpy", "matplotlib", "seaborn", | |
| "apache spark", "airflow", "mlflow", "weights & biases", | |
| # Databases | |
| "postgresql", "mysql", "mongodb", "redis", "elasticsearch", | |
| "cassandra", "dynamodb", "sqlite", "oracle", "mssql", | |
| "qdrant", "pinecone", "weaviate", "chroma", "faiss", | |
| # Cloud / DevOps | |
| "aws", "gcp", "azure", "docker", "kubernetes", "terraform", | |
| "github actions", "jenkins", "ci/cd", "lambda", "s3", "ec2", | |
| # Messaging / async | |
| "kafka", "rabbitmq", "celery", "sqs", "pubsub", | |
| # APIs / patterns | |
| "rest apis", "graphql", "grpc", "websockets", "microservices", | |
| "serverless", "api gateway", | |
| # Tools | |
| "git", "jira", "confluence", | |
| # Concepts | |
| "system design", "distributed systems", "agile", "tdd", | |
| "data structures", "algorithms", | |
| # Java ecosystem | |
| "springboot", "spring", "hibernate", "maven", "gradle", | |
| # .NET | |
| ".net", "asp.net", | |
| } | |
| def extract_skills_from_text( | |
| text: str, is_required: bool | |
| ) -> list[SkillRequirement]: | |
| """ | |
| Scan text for known tech skills and extract with proficiency context. | |
| Returns deduplicated list. | |
| """ | |
| lower = text.lower() | |
| found: dict[str, SkillRequirement] = {} | |
| for skill in sorted(TECH_VOCAB, key=len, reverse=True): | |
| if skill not in lower: | |
| continue | |
| # Find position and extract surrounding context (40 chars before) | |
| idx = lower.find(skill) | |
| context_before = lower[max(0, idx - 40): idx] | |
| context_after = lower[idx + len(skill): idx + len(skill) + 30] | |
| context = context_before + " " + context_after | |
| # Determine level from context qualifiers | |
| level = _level_from_context(context) | |
| normalized = normalize_skill_name(skill) | |
| if normalized not in found: | |
| found[normalized] = SkillRequirement( | |
| name=normalized, | |
| level=level, | |
| is_required=is_required, | |
| raw=skill, | |
| ) | |
| return list(found.values()) | |
| def _level_from_context(context: str) -> int: | |
| """Infer required proficiency from surrounding words.""" | |
| from app.pipeline.skill_parser import _QUALIFIER_MAP, _SORTED_QUALIFIERS | |
| lower = context.lower() | |
| for qualifier in _SORTED_QUALIFIERS: | |
| if qualifier in lower: | |
| return _QUALIFIER_MAP[qualifier] | |
| return 3 # default: intermediate | |
| # βββ Section extraction βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| _REQUIRED_HEADERS = [ | |
| "core requirements", "requirements", "required qualifications", | |
| "must have", "what you need", "you will need", "technical requirements", | |
| "skills required", | |
| ] | |
| _PREFERRED_HEADERS = [ | |
| "preferred qualifications", "preferred", "nice to have", | |
| "bonus points", "brownie points", "good to have", | |
| "what would be great", "plus", | |
| ] | |
| _RESPONSIBILITY_HEADERS = [ | |
| "responsibilities", "what you'll do", "what you will do", | |
| "your role", "duties", "key responsibilities", | |
| "role and responsibilities", | |
| ] | |
| def _extract_section(text: str, headers: list[str]) -> str: | |
| """Return text between the first matching header and the next section break.""" | |
| lower = text.lower() | |
| start = -1 | |
| for header in headers: | |
| idx = lower.find(header) | |
| if idx != -1: | |
| start = idx | |
| break | |
| if start == -1: | |
| return "" | |
| # Find next section break (another header or double newline after 50+ chars) | |
| end = len(text) | |
| for other_headers in [_REQUIRED_HEADERS, _PREFERRED_HEADERS, _RESPONSIBILITY_HEADERS]: | |
| for h in other_headers: | |
| idx = lower.find(h, start + 20) | |
| if idx != -1 and idx < end: | |
| end = idx | |
| return text[start:end] | |
| def _parse_bullets(section_text: str) -> list[str]: | |
| """Extract bullet points from a section.""" | |
| bullets: list[str] = [] | |
| for line in section_text.split("\n"): | |
| line = line.strip() | |
| # Remove bullet markers | |
| line = re.sub(r"^[β’Β·\-\*ββ€βΊβΈβͺβ¦β£β]\s*", "", line).strip() | |
| if len(line) > 5: | |
| bullets.append(line) | |
| return bullets | |
| # βββ Skills from "Skills Required" comma list βββββββββββββββββββββββββββββββββ | |
| def _parse_skills_section(text: str) -> list[str]: | |
| """Find 'Skills Required' or similar and return comma/newline split items.""" | |
| lower = text.lower() | |
| for header in ["skills required", "technical skills", "technologies", "key skills"]: | |
| idx = lower.find(header) | |
| if idx != -1: | |
| # Grab next 500 chars | |
| snippet = text[idx: idx + 500] | |
| # Split by comma or newline | |
| tokens = re.split(r"[,\n]", snippet) | |
| # Skip the header line itself | |
| result = [] | |
| for t in tokens[1:]: | |
| t = t.strip().strip("β’Β·-*β").strip() | |
| if 2 < len(t) < 60: | |
| result.append(t) | |
| return result | |
| return [] | |
| # βββ Main JD parser βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def parse_jd(text: str, jd_id: Optional[str] = None) -> JDRequirements: | |
| if jd_id is None: | |
| jd_id = str(uuid.uuid4())[:8] | |
| title = extract_title(text) | |
| engineer_type = infer_engineer_type(title, text) | |
| min_years = extract_min_years(text) | |
| # Extract sections | |
| req_section = _extract_section(text, _REQUIRED_HEADERS) | |
| pref_section = _extract_section(text, _PREFERRED_HEADERS) | |
| resp_section = _extract_section(text, _RESPONSIBILITY_HEADERS) | |
| # Parse skills from dedicated sections using vocab matching | |
| required_skills = extract_skills_from_text(req_section or text[:800], is_required=True) | |
| preferred_skills = extract_skills_from_text(pref_section, is_required=False) | |
| # Also parse comma-separated "Skills Required" section if present. | |
| # ONLY add tokens that match known TECH_VOCAB β prevents job titles ("AI Engineer"), | |
| # section headers ("Overview"), and sentence fragments ("and managing iot") from | |
| # leaking into the required skills list. | |
| _normalized_vocab = {normalize_skill_name(s) for s in TECH_VOCAB} | |
| comma_skills = _parse_skills_section(text) | |
| existing_names = {s.name for s in required_skills} | |
| for raw_skill in comma_skills: | |
| entry = parse_skill_token(raw_skill) | |
| if entry.name and entry.name not in existing_names and entry.name in _normalized_vocab: | |
| required_skills.append(SkillRequirement( | |
| name=entry.name, | |
| level=entry.level, | |
| is_required=True, | |
| raw=raw_skill, | |
| )) | |
| existing_names.add(entry.name) | |
| # Responsibilities as plain bullets | |
| responsibilities = _parse_bullets(resp_section)[:8] | |
| # Fallback: if we extracted 0 required skills, scan full text | |
| if not required_skills: | |
| required_skills = extract_skills_from_text(text, is_required=True) | |
| return JDRequirements( | |
| id=jd_id, | |
| title=title, | |
| engineer_type=engineer_type, | |
| min_years=min_years, | |
| required_skills=required_skills, | |
| preferred_skills=preferred_skills, | |
| responsibilities=responsibilities, | |
| raw_text=text[:200], | |
| full_text=text, | |
| ) | |