AnishDev commited on
Commit
ff7b177
Β·
0 Parent(s):

initial code and logic pushed

Browse files
.env.example ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ─── AWS Bedrock (for explanation generation) ───────────────────────────────
2
+ # IAM user needs: bedrock:InvokeModel permission
3
+ # Enable model access at: AWS Console β†’ Bedrock β†’ Model Access β†’ Claude 3.5 Haiku
4
+ AWS_ACCESS_KEY_ID=your_aws_access_key_here
5
+ AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key_here
6
+ AWS_DEFAULT_REGION=us-east-1
7
+
8
+ # ─── Redis ───────────────────────────────────────────────────────────────────
9
+ # Local Docker: redis://localhost:6379
10
+ # Fly.io Redis: redis://default:<password>@<host>:6379
11
+ REDIS_URL=redis://localhost:6379
12
+
13
+ # ─── CORS ────────────────────────────────────────────────────────────────────
14
+ # Comma-separated list of allowed frontend origins
15
+ CORS_ORIGINS=http://localhost:3000,https://your-frontend.vercel.app
16
+
17
+ # ─── App ─────────────────────────────────────────────────────────────────────
18
+ DEBUG=false
19
+ LOG_LEVEL=INFO
20
+
21
+ # ─── Embedding model (sentence-transformers, no API key needed) ───────────────
22
+ EMBEDDING_MODEL=all-MiniLM-L6-v2
23
+
24
+ # ─── Bedrock model ───────────────────────────────────────────────────────────
25
+ # Cheapest capable model; ~$0.002 per explanation
26
+ BEDROCK_MODEL_ID=anthropic.claude-3-5-haiku-20241022-v1:0
27
+
28
+ # ─── Matching thresholds ─────────────────────────────────────────────────────
29
+ # Candidate count above which BM25 pre-filter activates
30
+ BM25_PREFILTER_THRESHOLD=200
31
+ # How many candidates BM25 keeps before semantic scoring
32
+ BM25_TOP_K=500
.gitignore ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ .env
2
+ venv/
3
+ __pycache__/
4
+ *.pyc
5
+ *.pyo
6
+ .pytest_cache/
7
+ *.egg-info/
8
+ dist/
9
+ build/
Dockerfile ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Use a slim Python base image
2
+ FROM python:3.12-slim
3
+
4
+ # Set the working directory in the container
5
+ WORKDIR /app
6
+
7
+ # Install system dependencies required for PyMuPDF (fitz)
8
+ # This is in a separate layer to leverage Docker's build cache.
9
+ RUN apt-get update && apt-get install -y --no-install-recommends \
10
+ libmupdf-dev \
11
+ gcc \
12
+ && rm -rf /var/lib/apt/lists/*
13
+
14
+ # Copy the requirements file and install Python dependencies
15
+ COPY requirements.txt .
16
+ RUN pip install --no-cache-dir -r requirements.txt
17
+
18
+ # Pre-download and cache the sentence transformer model.
19
+ # This ensures the model is ready on first boot and avoids download delays.
20
+ RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')"
21
+
22
+ # Copy the application code into the container
23
+ COPY app/ ./app/
24
+
25
+ # Expose the port the app runs on
26
+ EXPOSE 8000
27
+
28
+ # Define the command to run the application
29
+ # Use --host 0.0.0.0 to make it accessible from outside the container
30
+ CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
README.md ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Coderound AI - Candidate Matching System
2
+
3
+ This repository contains the backend for the Coderound AI Candidate Matching System. It's a sophisticated pipeline designed to intelligently match candidates to job descriptions (JDs) using a hybrid AI approach.
4
+
5
+ ## Project Overview
6
+
7
+ The system is built to address the challenge of efficiently and accurately screening a large volume of candidates for technical roles. It automates the parsing of unstructured data from JD PDFs and candidate resumes (in XLSX format), evaluates their suitability, and provides AI-generated explanations for its matching decisions.
8
+
9
+ The core of the system is a multi-stage matching pipeline that combines traditional information retrieval techniques with modern NLP models:
10
+
11
+ 1. **Data Parsing:** Ingests and structures data from various formats. JDs are parsed from PDFs to extract key requirements, while candidate data is parsed from XLSX files.
12
+ 2. **BM25 Pre-filtering:** For large candidate pools (>200), the system first uses a BM25 relevance model to quickly identify a smaller, more relevant subset of candidates. This is a crucial first step for scalability.
13
+ 3. **Semantic Matching:** The filtered candidates are then processed through a semantic search pipeline using sentence transformers (`all-MiniLM-L6-v2`). This model, run locally, generates embeddings for candidate experience and JD requirements to find semantic similarities.
14
+ 4. **Proficiency-Aware Scoring:** The system doesn't just check for skill presence; it scores based on proficiency levels (e.g., "Expert" vs. "Familiar"), providing a more nuanced evaluation.
15
+ 5. **AI-Powered Explanations:** For top matches, the system can generate a detailed explanation of why a candidate is a good fit, using a large language model hosted on Ollama Cloud.
16
+
17
+ The entire process is exposed via a RESTful API built with FastAPI, featuring asynchronous processing for long-running matching tasks.
18
+
19
+ ## Tech Stack
20
+
21
+ | Category | Technology | Purpose |
22
+ | --- | --- | --- |
23
+ | **Framework** | FastAPI | High-performance Python web framework for building APIs. |
24
+ | **Server** | Uvicorn | ASGI server for running the FastAPI application. |
25
+ | **NLP & Matching** | Sentence-Transformers | For generating local sentence embeddings. |
26
+ | | Rank-BM25 | Efficient keyword-based pre-filtering. |
27
+ | | PyMuPDF | PDF parsing and text extraction. |
28
+ | | Pandas, Openpyxl | Parsing and handling XLSX candidate files. |
29
+ | **AI Explanations** | Ollama | For generating natural language explanations of matches. |
30
+ | **Caching & State** | Redis | Used for caching results and managing task status. |
31
+ | **Configuration** | Pydantic | For settings management and data validation. |
32
+
33
+ ## Scaling to 100,000 Candidates
34
+
35
+ The current architecture is designed with scalability in mind. Here’s how it would handle a load of 100,000 candidates and the changes that would be required to ensure performance.
36
+
37
+ ### Current Scalability Features:
38
+
39
+ * **Asynchronous Processing:** Matching jobs are run as background tasks, so the API remains responsive. The client can poll for results.
40
+ * **BM25 Pre-filtering:** The pipeline intelligently activates a BM25 pre-filter when the candidate pool exceeds 200. This significantly reduces the number of candidates that need to go through the more computationally expensive embedding and scoring stages.
41
+ * **Efficient Embeddings:** Using a lightweight local model like `all-MiniLM-L6-v2` avoids network latency and API costs associated with external embedding services.
42
+
43
+ ### Architectural Changes for 100k+ Scale:
44
+
45
+ 1. **Distributed Task Queue:**
46
+ * **Current:** FastAPI's `BackgroundTasks` is suitable for a single-server setup.
47
+ * **Proposed Change:** Replace `BackgroundTasks` with a dedicated distributed task queue like **Celery** with a **RabbitMQ** or **Redis** broker. This would allow us to distribute matching jobs across multiple worker nodes, scaling horizontally.
48
+
49
+ 2. **Vector Database:**
50
+ * **Current:** Embeddings are generated on-the-fly and stored in memory or a simple cache.
51
+ * **Proposed Change:** Pre-compute and store candidate embeddings in a dedicated **vector database** like **Pinecone, Weaviate, or Milvus**. This would transform the matching process from a "search" to a "lookup," drastically reducing latency. The workflow would be:
52
+ 1. When a candidate is uploaded, their profile is parsed, embedded, and stored in the vector DB.
53
+ 2. When a JD is uploaded, its requirements are embedded.
54
+ 3. The JD embedding is used to query the vector DB to find the top-K most similar candidates in milliseconds.
55
+
56
+ 3. **Dedicated Microservices:**
57
+ * **Current:** The application is a monolith.
58
+ * **Proposed Change:** Break down the application into microservices:
59
+ * `Parser Service`: Handles file parsing (PDF, XLSX).
60
+ * `Matching Service`: Contains the core matching logic and workers.
61
+ * `API Gateway`: The main entry point for all requests.
62
+ * `Explanation Service`: Manages interaction with the Ollama LLM.
63
+ This would allow independent scaling of each component. For example, we could scale up the `Matching Service` workers during peak hiring seasons without touching other services.
64
+
65
+ 4. **Container Orchestration:**
66
+ * **Current:** The app is containerized with Docker and can be deployed on a platform like Fly.io.
67
+ * **Proposed Change:** Use **Kubernetes** to manage the containerized microservices. Kubernetes would handle auto-scaling, service discovery, and load balancing automatically.
68
+
69
+ ### Diagram of Scaled Architecture
70
+
71
+ ```mermaid
72
+ graph TD
73
+ subgraph "User"
74
+ A[Recruiter]
75
+ end
76
+
77
+ subgraph "API Layer"
78
+ B[API Gateway]
79
+ end
80
+
81
+ subgraph "Services"
82
+ C[Parser Service]
83
+ D[Matching Service]
84
+ E[Explanation Service]
85
+ end
86
+
87
+ subgraph "Data & AI Layer"
88
+ F[Vector Database e.g., Pinecone]
89
+ G[Task Queue e.g., Celery + RabbitMQ]
90
+ H[Redis Cache]
91
+ I[Ollama LLM]
92
+ end
93
+
94
+ A --> B
95
+ B --> C
96
+ B --> D
97
+ B --> E
98
+
99
+ C --> G
100
+ D -- Reads from --> F
101
+ D -- Uses --> G
102
+ D -- Caches in --> H
103
+ E -- Calls --> I
104
+ ```
105
+
106
+ ## Sample Data for Testing
107
+
108
+ ### Sample Job Description
109
+
110
+ **Role:** Senior Backend Engineer (Python)
111
+ **Experience:** 5+ years
112
+ **Required Skills:**
113
+ * Python (Expert)
114
+ * Django or FastAPI (Expert)
115
+ * PostgreSQL (Advanced)
116
+ * Redis (Advanced)
117
+ * Docker (Advanced)
118
+ * Celery (Intermediate)
119
+
120
+ **Preferred Skills:**
121
+ * Kubernetes (Intermediate)
122
+ * Terraform (Familiar)
123
+ * AWS or GCP (Advanced)
124
+
125
+ ### Sample Candidate Profiles
126
+
127
+ **Candidate 1: Strong Match**
128
+ * **Experience:** 7 years as a Backend Engineer.
129
+ * **Skills:** Python (Expert), FastAPI (Expert), PostgreSQL (Advanced), Redis (Expert), Docker (Advanced), Kubernetes (Intermediate), AWS (Advanced).
130
+ * **Summary:** Built and scaled a microservices-based system using FastAPI, deployed on AWS with Kubernetes.
131
+
132
+ **Candidate 2: Medium Match**
133
+ * **Experience:** 4 years as a Full-Stack Developer.
134
+ * **Skills:** Python (Advanced), Django (Advanced), MySQL (Advanced), Docker (Intermediate), Heroku (Advanced).
135
+ * **Summary:** Developed monolithic web applications with Django. Has some containerization experience but lacks cloud-native and distributed systems knowledge.
136
+
137
+ **Candidate 3: Low Match**
138
+ * **Experience:** 2 years as a Data Analyst.
139
+ * **Skills:** Python (Intermediate), Pandas (Expert), SQL (Advanced), Tableau (Expert).
140
+ * **Summary:** Focused on data analysis and visualization. Limited backend development experience.
141
+
142
+ ## Setup and Deployment
143
+
144
+ Follow the instructions below to set up the project locally or deploy it.
145
+
146
+ ### Local Setup
147
+
148
+ 1. **Prerequisites:** Python 3.11+
149
+ 2. **Install dependencies:**
150
+ ```bash
151
+ pip install -r requirements.txt
152
+ ```
153
+ 3. **Configure Environment:**
154
+ Create a `.env` file from the example and fill in your details.
155
+ ```bash
156
+ cp .env.example .env
157
+ ```
158
+ 4. **Run the server:**
159
+ ```bash
160
+ uvicorn app.main:app --reload --port 8000
161
+ ```
162
+
163
+ ### Deployment (Fly.io)
164
+
165
+ This application is configured for deployment on [Fly.io](https://fly.io/).
166
+
167
+ 1. **Login to Fly:**
168
+ ```bash
169
+ fly auth login
170
+ ```
171
+ 2. **Set Secrets:**
172
+ The application requires environment variables to be set as secrets on Fly.io.
173
+ ```bash
174
+ fly secrets set CORS_ORIGINS="<your_frontend_url>"
175
+ fly secrets set REDIS_URL="<your_fly_redis_url>"
176
+ # Optional: for AI explanations
177
+ fly secrets set OLLAMA_HOST="<your_ollama_host>" OLLAMA_API_KEY="<your_key>"
178
+ ```
179
+ 3. **Deploy:**
180
+ ```bash
181
+ fly deploy
182
+ ```
183
+
184
+
185
+ | Variable | Required | Description | Example |
186
+ |----------|----------|-------------|---------|
187
+ | `OLLAMA_HOST` | No | Ollama server URL | `https://ollama.com` |
188
+ | `OLLAMA_API_KEY` | No | Ollama cloud API key | `abc123...` |
189
+ | `OLLAMA_MODEL` | No | Model name | `gpt-oss:120b-cloud` |
190
+ | `REDIS_URL` | No | Redis connection string (use `rediss://` for TLS) | `rediss://default:pass@host:port` |
191
+ | `CORS_ORIGINS` | Yes | Comma-separated allowed origins | `http://localhost:3000` |
192
+ | `EMBEDDING_MODEL` | No | sentence-transformers model | `all-MiniLM-L6-v2` |
193
+ | `BM25_PREFILTER_THRESHOLD` | No | Min candidates before BM25 activates | `200` |
194
+ | `BM25_TOP_K` | No | Candidates kept after BM25 | `500` |
195
+ | `LOG_LEVEL` | No | Logging level | `INFO` |
196
+
197
+ > **Redis URL**: Upstash requires `rediss://` (double-s) for TLS. Using `redis://` will fail with SSL EOF errors.
198
+
199
+ ## API Reference
200
+
201
+ ```
202
+ GET /health
203
+ β†’ {status, redis, embedding_model_loaded}
204
+
205
+ POST /api/v1/jd/upload multipart: file=<PDF>
206
+ β†’ {jd_id, title, engineer_type, min_years, required_skills_count}
207
+
208
+ GET /api/v1/jd/
209
+ β†’ [{jd_id, title, engineer_type, uploaded_at}, ...]
210
+
211
+ GET /api/v1/jd/{jd_id}
212
+ β†’ full JD with skills and responsibilities
213
+
214
+ POST /api/v1/candidates/upload multipart: file=<XLSX>
215
+ β†’ {batch_id, total_count, engineer_type_distribution}
216
+
217
+ GET /api/v1/candidates/
218
+ β†’ [{batch_id, total_count, uploaded_at}, ...]
219
+
220
+ POST /api/v1/match/{jd_id} body: {batch_id, top_n}
221
+ β†’ {match_id, status: "pending"} 202 Accepted
222
+
223
+ GET /api/v1/status/{match_id}
224
+ β†’ {status: "pending"|"running"|"completed"|"failed", progress, message}
225
+
226
+ GET /api/v1/match/{jd_id}?batch_id=&top_n=
227
+ β†’ [{rank, candidate_id, name, final_score, tier, matched_skills,
228
+ missing_skills, below_level_skills, flags, ...}, ...]
229
+
230
+ GET /api/v1/match/{jd_id}/{candidate_id}?batch_id=&generate=true
231
+ β†’ full score breakdown + AI explanation (generate=true triggers Ollama)
232
+ ```
233
+
234
+ ## Scoring Pipeline
235
+
236
+ ```
237
+ Hard filter (engineer type + years of experience)
238
+ ↓
239
+ BM25 pre-filter (only if candidates > 200 β†’ keeps top 500)
240
+ ↓
241
+ Local embeddings (all-MiniLM-L6-v2, candidate work history only)
242
+ ↓
243
+ Per-candidate score:
244
+ 35% β€” proficiency-aware skill matching
245
+ 40% β€” semantic similarity (JD vs work history)
246
+ 10% β€” position title alignment
247
+ 15% β€” domain score (GenAI depth for AI JDs, stack for Backend)
248
+ ↓
249
+ Penalties applied (type mismatch, years gap, missing fields)
250
+ ↓
251
+ Ranked β†’ Redis ZSET β†’ top-N retrieval O(log N)
252
+ ```
253
+
254
+ ## XLSX Format
255
+
256
+ Required columns:
257
+
258
+ | Column | Description |
259
+ |--------|-------------|
260
+ | `name` | Full name |
261
+ | `current_position` | Job title / role |
262
+ | `engineer_type` | "AI Engineer", "Backend Engineer", etc. |
263
+ | `parsed_skills` | Skills text |
264
+ | `parsed_summary` | Professional summary |
265
+ | `parsed_work_experience` | Stringified Python list of `{company, role, duration, description}` dicts |
266
+ | `parsed_metadata_calculated_years_experience` | Years of experience (float) |
267
+
268
+ Optional: `gen_ai_experience`, `current_title`, `is_actively_or_passively_looking`, `notice_period`, `parsed_metadata_most_recent_company`
269
+
270
+ ## Deployment
271
+
272
+ ### Docker
273
+
274
+ ```bash
275
+ docker build -t jd-matcher-backend .
276
+ docker run -p 8000:8000 --env-file .env jd-matcher-backend
277
+ ```
278
+
279
+ ### Fly.io
280
+
281
+ ```bash
282
+ fly launch
283
+ fly secrets set OLLAMA_API_KEY=xxx
284
+ fly secrets set REDIS_URL=rediss://...
285
+ fly secrets set CORS_ORIGINS=https://your-frontend.vercel.app
286
+ fly deploy
287
+ ```
288
+
289
+ See `fly.toml` for machine sizing (2GB RAM recommended for embedding model).
290
+
291
+ ## Storage Notes
292
+
293
+ Candidate JSON is stored in **process memory**, not Redis. This avoids the Upstash free tier 10MB value limit. Process restart clears candidates β€” re-upload required. All match results and metadata are in Redis with 24-48h TTL.
294
+
295
+ If Redis is unavailable, storage auto-falls back to `_MemStore` (in-process dict with TTL). System continues working without any external dependencies.
app/__init__.py ADDED
File without changes
app/api/__init__.py ADDED
File without changes
app/api/routes/__init__.py ADDED
File without changes
app/api/routes/candidates.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import uuid
2
+ import json
3
+ import logging
4
+ from datetime import datetime, timezone
5
+ from dataclasses import asdict
6
+
7
+ from fastapi import APIRouter, UploadFile, File, HTTPException
8
+
9
+ from app.services import storage, file_parser
10
+ from app.models.schemas import CandidateBatchResponse, BatchListItem
11
+
12
+ router = APIRouter(prefix="/api/v1/candidates", tags=["candidates"])
13
+ logger = logging.getLogger(__name__)
14
+
15
+
16
+ def _candidate_to_dict(c) -> dict:
17
+ """Serialize CandidateData to JSON-safe dict."""
18
+ return {
19
+ "id": c.id,
20
+ "name": c.name,
21
+ "current_position": c.current_position,
22
+ "current_title": c.current_title,
23
+ "years_of_experience": c.years_of_experience,
24
+ "engineer_type": c.engineer_type,
25
+ "gen_ai_experience": c.gen_ai_experience,
26
+ "parsed_skills": c.parsed_skills,
27
+ "parsed_summary": c.parsed_summary,
28
+ "parsed_work_experience_raw": c.parsed_work_experience_raw,
29
+ "parsed_work_experience_text": c.parsed_work_experience_text,
30
+ "work_experiences": [
31
+ {
32
+ "company": we.company,
33
+ "role": we.role,
34
+ "duration": we.duration,
35
+ "description": we.description,
36
+ }
37
+ for we in c.work_experiences
38
+ ],
39
+ "most_recent_company": c.most_recent_company,
40
+ "notice_period": c.notice_period,
41
+ "is_actively_looking": c.is_actively_looking,
42
+ "flags": c.flags,
43
+ "penalties": c.penalties,
44
+ }
45
+
46
+
47
+ @router.post("/upload", response_model=CandidateBatchResponse)
48
+ async def upload_candidates(file: UploadFile = File(...)):
49
+ if not file.filename or not file.filename.lower().endswith(".xlsx"):
50
+ raise HTTPException(400, "Only XLSX files accepted for candidates")
51
+
52
+ content = await file.read()
53
+ if len(content) > 50 * 1024 * 1024:
54
+ raise HTTPException(413, "File too large (max 50 MB)")
55
+
56
+ try:
57
+ candidates, dist = file_parser.parse_xlsx(content)
58
+ except ValueError as e:
59
+ raise HTTPException(400, str(e))
60
+
61
+ if not candidates:
62
+ raise HTTPException(400, "No valid candidates found in file")
63
+
64
+ batch_id = str(uuid.uuid4())[:8]
65
+ now = datetime.now(timezone.utc).isoformat()
66
+
67
+ metadata = {
68
+ "batch_id": batch_id,
69
+ "total_count": len(candidates),
70
+ "engineer_type_distribution": json.dumps(dist),
71
+ "uploaded_at": now,
72
+ "filename": file.filename,
73
+ }
74
+
75
+ candidates_json = json.dumps([_candidate_to_dict(c) for c in candidates])
76
+ storage.save_batch(batch_id, metadata, candidates_json)
77
+
78
+ logger.info(f"Batch uploaded: {batch_id}, {len(candidates)} candidates, dist={dist}")
79
+ return CandidateBatchResponse(
80
+ batch_id=batch_id,
81
+ total_count=len(candidates),
82
+ engineer_type_distribution=dist,
83
+ uploaded_at=now,
84
+ )
85
+
86
+
87
+ @router.get("/", response_model=list[BatchListItem])
88
+ def list_batches():
89
+ items = storage.list_batches()
90
+ result = []
91
+ for d in items:
92
+ dist = d.get("engineer_type_distribution", {})
93
+ if isinstance(dist, str):
94
+ try:
95
+ dist = json.loads(dist)
96
+ except Exception:
97
+ dist = {}
98
+ result.append(BatchListItem(
99
+ batch_id=d.get("batch_id", ""),
100
+ total_count=int(d.get("total_count", 0)),
101
+ engineer_type_distribution=dist,
102
+ uploaded_at=d.get("uploaded_at", ""),
103
+ ))
104
+ return result
app/api/routes/health.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter
2
+ from app.services import storage
3
+
4
+ router = APIRouter(tags=["health"])
5
+
6
+
7
+ @router.get("/health")
8
+ def health_check():
9
+ redis_ok = storage.ping()
10
+ return {
11
+ "status": "ok" if redis_ok else "degraded",
12
+ "redis": "connected" if redis_ok else "disconnected",
13
+ }
app/api/routes/jd.py ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import uuid
2
+ import json
3
+ import logging
4
+ from datetime import datetime, timezone
5
+
6
+ from fastapi import APIRouter, UploadFile, File, HTTPException
7
+
8
+ from app.services import storage, file_parser
9
+ from app.pipeline.jd_parser import parse_jd
10
+ from app.models.schemas import JDUploadResponse, JDListItem, JDDetailResponse
11
+
12
+ router = APIRouter(prefix="/api/v1/jd", tags=["jd"])
13
+ logger = logging.getLogger(__name__)
14
+
15
+
16
+ @router.post("/upload", response_model=JDUploadResponse)
17
+ async def upload_jd(file: UploadFile = File(...)):
18
+ if not file.filename or not file.filename.lower().endswith(".pdf"):
19
+ raise HTTPException(400, "Only PDF files accepted for JDs")
20
+
21
+ content = await file.read()
22
+ if len(content) > 10 * 1024 * 1024:
23
+ raise HTTPException(413, "PDF too large (max 10 MB)")
24
+
25
+ try:
26
+ text = file_parser.parse_pdf(content)
27
+ except ValueError as e:
28
+ raise HTTPException(400, str(e))
29
+
30
+ jd_id = str(uuid.uuid4())[:8]
31
+ jd = parse_jd(text, jd_id=jd_id)
32
+
33
+ now = datetime.now(timezone.utc).isoformat()
34
+ data = {
35
+ "jd_id": jd_id,
36
+ "title": jd.title,
37
+ "engineer_type": jd.engineer_type or "",
38
+ "min_years": jd.min_years,
39
+ "required_skills": json.dumps([
40
+ {"name": s.name, "level": s.level, "raw": s.raw}
41
+ for s in jd.required_skills
42
+ ]),
43
+ "preferred_skills": json.dumps([
44
+ {"name": s.name, "level": s.level, "raw": s.raw}
45
+ for s in jd.preferred_skills
46
+ ]),
47
+ "responsibilities": json.dumps(jd.responsibilities),
48
+ "full_text": jd.full_text[:3000],
49
+ "uploaded_at": now,
50
+ }
51
+ storage.save_jd(jd_id, data)
52
+
53
+ logger.info(f"JD uploaded: {jd_id} '{jd.title}'")
54
+ return JDUploadResponse(
55
+ jd_id=jd_id,
56
+ title=jd.title,
57
+ engineer_type=jd.engineer_type,
58
+ min_years=jd.min_years,
59
+ required_skills_count=len(jd.required_skills),
60
+ preferred_skills_count=len(jd.preferred_skills),
61
+ uploaded_at=now,
62
+ )
63
+
64
+
65
+ @router.get("/", response_model=list[JDListItem])
66
+ def list_jds():
67
+ items = storage.list_jds()
68
+ result = []
69
+ for d in items:
70
+ try:
71
+ req = json.loads(d.get("required_skills", "[]"))
72
+ except Exception:
73
+ req = []
74
+ result.append(JDListItem(
75
+ jd_id=d.get("jd_id", ""),
76
+ title=d.get("title", ""),
77
+ engineer_type=d.get("engineer_type") or None,
78
+ min_years=float(d.get("min_years", 0)),
79
+ required_skills_count=len(req),
80
+ uploaded_at=d.get("uploaded_at", ""),
81
+ ))
82
+ return result
83
+
84
+
85
+ @router.get("/{jd_id}", response_model=JDDetailResponse)
86
+ def get_jd(jd_id: str):
87
+ data = storage.get_jd(jd_id)
88
+ if not data:
89
+ raise HTTPException(404, f"JD {jd_id} not found")
90
+
91
+ req = data.get("required_skills", [])
92
+ pref = data.get("preferred_skills", [])
93
+ resp = data.get("responsibilities", [])
94
+
95
+ if isinstance(req, str):
96
+ req = json.loads(req)
97
+ if isinstance(pref, str):
98
+ pref = json.loads(pref)
99
+ if isinstance(resp, str):
100
+ resp = json.loads(resp)
101
+
102
+ return JDDetailResponse(
103
+ jd_id=data.get("jd_id", jd_id),
104
+ title=data.get("title", ""),
105
+ engineer_type=data.get("engineer_type") or None,
106
+ min_years=float(data.get("min_years", 0)),
107
+ required_skills_count=len(req),
108
+ uploaded_at=data.get("uploaded_at", ""),
109
+ required_skills=req,
110
+ preferred_skills=pref,
111
+ responsibilities=resp,
112
+ )
app/api/routes/match.py ADDED
@@ -0,0 +1,474 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import uuid
2
+ import json
3
+ import logging
4
+ from typing import Optional
5
+
6
+ from fastapi import APIRouter, HTTPException, BackgroundTasks, Query
7
+
8
+ from app.services import storage, explainer
9
+ from app.pipeline.matcher import run_matching
10
+ from app.pipeline.skill_parser import parse_skills_string, upgrade_levels_from_work_exp
11
+ from app.models.domain import (
12
+ CandidateData, WorkExperience, JDRequirements, SkillRequirement
13
+ )
14
+ from app.models.schemas import (
15
+ MatchTriggerRequest, MatchTriggerResponse,
16
+ MatchResultsResponse, CandidateMatchResponse, CandidateDetailResponse,
17
+ ScoreBreakdownResponse, SkillMatchInfo, StatusResponse,
18
+ )
19
+
20
+ router = APIRouter(prefix="/api/v1/match", tags=["match"])
21
+
22
+
23
+ # NOTE: /status/{match_id} MUST be declared before /{jd_id} to avoid route
24
+ # shadowing β€” FastAPI matches routes in declaration order.
25
+ logger = logging.getLogger(__name__)
26
+
27
+
28
+ # ─── Deserializers ────────────────────────────────────────────────────────────
29
+
30
+ def _dict_to_candidate(d: dict) -> CandidateData:
31
+ work_exps = [
32
+ WorkExperience(
33
+ company=we.get("company", ""),
34
+ role=we.get("role", ""),
35
+ duration=we.get("duration", ""),
36
+ description=we.get("description", ""),
37
+ )
38
+ for we in d.get("work_experiences", [])
39
+ ]
40
+ c = CandidateData(
41
+ id=d["id"],
42
+ name=d["name"],
43
+ current_position=d.get("current_position", ""),
44
+ current_title=d.get("current_title"),
45
+ years_of_experience=float(d.get("years_of_experience", -1)),
46
+ engineer_type=d.get("engineer_type", "Unknown"),
47
+ gen_ai_experience=d.get("gen_ai_experience"),
48
+ parsed_skills=d.get("parsed_skills", ""),
49
+ parsed_summary=d.get("parsed_summary", ""),
50
+ parsed_work_experience_raw=d.get("parsed_work_experience_raw", ""),
51
+ parsed_work_experience_text=d.get("parsed_work_experience_text", ""),
52
+ work_experiences=work_exps,
53
+ extracted_skills=[],
54
+ most_recent_company=d.get("most_recent_company"),
55
+ notice_period=d.get("notice_period"),
56
+ is_actively_looking=d.get("is_actively_looking"),
57
+ flags=list(d.get("flags", [])),
58
+ penalties=list(d.get("penalties", [])),
59
+ )
60
+ return c
61
+
62
+
63
+ def _dict_to_jd(d: dict) -> JDRequirements:
64
+ def parse_skills(raw, is_req) -> list[SkillRequirement]:
65
+ if isinstance(raw, str):
66
+ try:
67
+ raw = json.loads(raw)
68
+ except Exception:
69
+ return []
70
+ result = []
71
+ for s in raw:
72
+ if isinstance(s, dict):
73
+ result.append(SkillRequirement(
74
+ name=s.get("name", ""),
75
+ level=int(s.get("level", 3)),
76
+ is_required=is_req,
77
+ raw=s.get("raw", ""),
78
+ ))
79
+ return result
80
+
81
+ req = parse_skills(d.get("required_skills", "[]"), True)
82
+ pref = parse_skills(d.get("preferred_skills", "[]"), False)
83
+ resp = d.get("responsibilities", [])
84
+ if isinstance(resp, str):
85
+ try:
86
+ resp = json.loads(resp)
87
+ except Exception:
88
+ resp = []
89
+
90
+ return JDRequirements(
91
+ id=d.get("jd_id", ""),
92
+ title=d.get("title", ""),
93
+ engineer_type=d.get("engineer_type") or None,
94
+ min_years=float(d.get("min_years", 0)),
95
+ required_skills=req,
96
+ preferred_skills=pref,
97
+ responsibilities=resp,
98
+ raw_text=d.get("full_text", "")[:200],
99
+ full_text=d.get("full_text", ""),
100
+ )
101
+
102
+
103
+ def _result_to_response(r) -> CandidateMatchResponse:
104
+ bd = r.score_breakdown
105
+ return CandidateMatchResponse(
106
+ rank=r.rank,
107
+ candidate_id=r.candidate_id,
108
+ name=r.candidate_name,
109
+ current_position=r.current_position,
110
+ years_of_experience=r.years_of_experience,
111
+ engineer_type=r.engineer_type,
112
+ final_score=round(r.final_score, 4),
113
+ tier=r.tier.value,
114
+ score_breakdown=ScoreBreakdownResponse(
115
+ skill_score=bd.skill_score,
116
+ semantic_score=bd.semantic_score,
117
+ position_score=bd.position_score,
118
+ special_score=bd.special_score,
119
+ final_score=bd.final_score,
120
+ penalties=bd.penalties_applied,
121
+ ),
122
+ matched_skills=[
123
+ SkillMatchInfo(
124
+ skill=s.skill,
125
+ candidate_level=s.candidate_level,
126
+ required_level=s.required_level,
127
+ score=s.score,
128
+ status=s.status,
129
+ )
130
+ for s in bd.matched_skills
131
+ ],
132
+ below_level_skills=[
133
+ SkillMatchInfo(
134
+ skill=s.skill,
135
+ candidate_level=s.candidate_level,
136
+ required_level=s.required_level,
137
+ score=s.score,
138
+ status=s.status,
139
+ )
140
+ for s in bd.below_level_skills
141
+ ],
142
+ missing_skills=bd.missing_skills,
143
+ flags=r.flags,
144
+ notice_period=r.notice_period,
145
+ most_recent_company=r.most_recent_company,
146
+ )
147
+
148
+
149
+ def _result_to_storage_dict(r, candidate: CandidateData) -> dict:
150
+ """Dict for detail hash β€” includes context needed for explanation."""
151
+ bd = r.score_breakdown
152
+ return {
153
+ "candidate_id": r.candidate_id,
154
+ "candidate_name": r.candidate_name,
155
+ "current_position": r.current_position,
156
+ "years_of_experience": r.years_of_experience,
157
+ "engineer_type": r.engineer_type,
158
+ "final_score": r.final_score,
159
+ "tier": r.tier.value,
160
+ "rank": r.rank,
161
+ "flags": json.dumps(r.flags),
162
+ "notice_period": r.notice_period or "",
163
+ "most_recent_company": r.most_recent_company or "",
164
+ "score_breakdown": json.dumps({
165
+ "skill_score": bd.skill_score,
166
+ "semantic_score": bd.semantic_score,
167
+ "position_score": bd.position_score,
168
+ "special_score": bd.special_score,
169
+ "final_score": bd.final_score,
170
+ "penalties": bd.penalties_applied,
171
+ "matched_skills": [
172
+ {"skill": s.skill, "candidate_level": s.candidate_level,
173
+ "required_level": s.required_level, "score": s.score, "status": s.status}
174
+ for s in bd.matched_skills
175
+ ],
176
+ "below_level_skills": [
177
+ {"skill": s.skill, "candidate_level": s.candidate_level,
178
+ "required_level": s.required_level, "score": s.score, "status": s.status}
179
+ for s in bd.below_level_skills
180
+ ],
181
+ "missing_skills": bd.missing_skills,
182
+ }),
183
+ # Context for LLM explanation (truncated)
184
+ "parsed_skills": candidate.parsed_skills[:500],
185
+ "parsed_summary": candidate.parsed_summary[:300],
186
+ "parsed_work_experience_text": candidate.parsed_work_experience_text[:600],
187
+ }
188
+
189
+
190
+ # ─── Match background task ────────────────────────────────────────────────────
191
+
192
+ def _run_match_task(
193
+ match_id: str,
194
+ jd_id: str,
195
+ batch_id: str,
196
+ top_n: int,
197
+ ):
198
+ """Runs in background. Updates match status in Redis."""
199
+ try:
200
+ storage.set_match_status(match_id, "processing", progress=0.1)
201
+
202
+ jd_data = storage.get_jd(jd_id)
203
+ if not jd_data:
204
+ storage.set_match_status(match_id, "failed", message=f"JD {jd_id} not found")
205
+ return
206
+
207
+ candidates_json = storage.get_batch_candidates_json(batch_id)
208
+ if not candidates_json:
209
+ storage.set_match_status(match_id, "failed", message=f"Batch {batch_id} not found")
210
+ return
211
+
212
+ jd = _dict_to_jd(jd_data)
213
+ raw_candidates = json.loads(candidates_json)
214
+ candidates = [_dict_to_candidate(d) for d in raw_candidates]
215
+
216
+ storage.set_match_status(match_id, "processing", progress=0.3)
217
+
218
+ results = run_matching(jd, candidates, top_n=top_n)
219
+
220
+ storage.set_match_status(match_id, "processing", progress=0.9)
221
+
222
+ # Save to Redis
223
+ storage_dicts = []
224
+ cand_map = {c.id: c for c in candidates}
225
+ for r in results:
226
+ cand = cand_map.get(r.candidate_id)
227
+ if cand:
228
+ storage_dicts.append(_result_to_storage_dict(r, cand))
229
+
230
+ storage.save_match_results(jd_id, batch_id, storage_dicts)
231
+ storage.set_active_batch(jd_id, batch_id)
232
+ storage.set_match_status(match_id, "completed", progress=1.0)
233
+
234
+ logger.info(f"Match {match_id} completed: {len(results)} results")
235
+
236
+ except Exception as e:
237
+ logger.exception(f"Match {match_id} failed")
238
+ storage.set_match_status(match_id, "failed", message=str(e))
239
+
240
+
241
+ # ─── Endpoints ────────────────────────────────────────────────────────────────
242
+
243
+ @router.post("/{jd_id}", response_model=MatchTriggerResponse)
244
+ async def trigger_match(
245
+ jd_id: str,
246
+ body: MatchTriggerRequest,
247
+ background_tasks: BackgroundTasks,
248
+ ):
249
+ jd_data = storage.get_jd(jd_id)
250
+ if not jd_data:
251
+ raise HTTPException(404, f"JD {jd_id} not found")
252
+
253
+ batch_meta = storage.get_batch_metadata(body.batch_id)
254
+ if not batch_meta:
255
+ raise HTTPException(404, f"Batch {body.batch_id} not found")
256
+
257
+ total = int(batch_meta.get("total_count", 0))
258
+ match_id = f"{jd_id}_{body.batch_id}"
259
+ top_n = body.top_n
260
+
261
+ storage.set_match_status(match_id, "pending")
262
+ background_tasks.add_task(_run_match_task, match_id, jd_id, body.batch_id, top_n)
263
+
264
+ return MatchTriggerResponse(
265
+ match_id=match_id,
266
+ status="pending",
267
+ total_candidates=total,
268
+ filtered_candidates=total,
269
+ )
270
+
271
+
272
+ @router.get("/{jd_id}", response_model=MatchResultsResponse)
273
+ def get_match_results(
274
+ jd_id: str,
275
+ batch_id: Optional[str] = Query(default=None),
276
+ top_n: int = Query(default=10, ge=1, le=50),
277
+ ):
278
+ if not batch_id:
279
+ batch_id = storage.get_active_batch(jd_id)
280
+ if not batch_id:
281
+ raise HTTPException(404, "No match results yet. Run POST /match/{jd_id} first.")
282
+
283
+ jd_data = storage.get_jd(jd_id)
284
+ if not jd_data:
285
+ raise HTTPException(404, f"JD {jd_id} not found")
286
+
287
+ ranked = storage.get_ranked_candidates(jd_id, batch_id, top_n)
288
+ if not ranked:
289
+ raise HTTPException(404, "No results found. Match may still be processing.")
290
+
291
+ candidates_out: list[CandidateMatchResponse] = []
292
+ for rank_i, (cid, score) in enumerate(ranked, start=1):
293
+ detail = storage.get_candidate_detail(jd_id, batch_id, cid)
294
+ if not detail:
295
+ continue
296
+
297
+ bd_raw = detail.get("score_breakdown", "{}")
298
+ if isinstance(bd_raw, str):
299
+ try:
300
+ bd_raw = json.loads(bd_raw)
301
+ except Exception:
302
+ bd_raw = {}
303
+
304
+ flags = detail.get("flags", [])
305
+ if isinstance(flags, str):
306
+ try:
307
+ flags = json.loads(flags)
308
+ except Exception:
309
+ flags = []
310
+
311
+ def _parse_skill_list(raw):
312
+ if isinstance(raw, str):
313
+ try:
314
+ raw = json.loads(raw)
315
+ except Exception:
316
+ return []
317
+ return raw or []
318
+
319
+ matched = [
320
+ SkillMatchInfo(**s) for s in _parse_skill_list(bd_raw.get("matched_skills", []))
321
+ ]
322
+ below = [
323
+ SkillMatchInfo(**s) for s in _parse_skill_list(bd_raw.get("below_level_skills", []))
324
+ ]
325
+
326
+ candidates_out.append(CandidateMatchResponse(
327
+ rank=rank_i,
328
+ candidate_id=cid,
329
+ name=detail.get("candidate_name", ""),
330
+ current_position=detail.get("current_position", ""),
331
+ years_of_experience=float(detail.get("years_of_experience", 0)),
332
+ engineer_type=detail.get("engineer_type", ""),
333
+ final_score=float(detail.get("final_score", score)),
334
+ tier=detail.get("tier", "Partial Match"),
335
+ score_breakdown=ScoreBreakdownResponse(
336
+ skill_score=float(bd_raw.get("skill_score", 0)),
337
+ semantic_score=float(bd_raw.get("semantic_score", 0)),
338
+ position_score=float(bd_raw.get("position_score", 0)),
339
+ special_score=float(bd_raw.get("special_score", 0)),
340
+ final_score=float(bd_raw.get("final_score", score)),
341
+ penalties=bd_raw.get("penalties", []),
342
+ ),
343
+ matched_skills=matched,
344
+ below_level_skills=below,
345
+ missing_skills=_parse_skill_list(bd_raw.get("missing_skills", [])),
346
+ flags=flags,
347
+ notice_period=detail.get("notice_period") or None,
348
+ most_recent_company=detail.get("most_recent_company") or None,
349
+ ))
350
+
351
+ return MatchResultsResponse(
352
+ jd_id=jd_id,
353
+ jd_title=jd_data.get("title", ""),
354
+ batch_id=batch_id,
355
+ top_n=top_n,
356
+ total_matched=len(candidates_out),
357
+ candidates=candidates_out,
358
+ )
359
+
360
+
361
+ @router.get("/{jd_id}/{candidate_id}", response_model=CandidateDetailResponse)
362
+ def get_candidate_detail(
363
+ jd_id: str,
364
+ candidate_id: str,
365
+ batch_id: Optional[str] = Query(default=None),
366
+ generate: bool = Query(default=False),
367
+ ):
368
+ if not batch_id:
369
+ batch_id = storage.get_active_batch(jd_id)
370
+ if not batch_id:
371
+ raise HTTPException(404, "No match results for this JD yet.")
372
+
373
+ detail = storage.get_candidate_detail(jd_id, batch_id, candidate_id)
374
+ if not detail:
375
+ raise HTTPException(404, f"Candidate {candidate_id} not found in results")
376
+
377
+ jd_data = storage.get_jd(jd_id)
378
+
379
+ # Check explanation cache
380
+ cached_explanation = storage.get_explanation(jd_id, batch_id, candidate_id)
381
+ explanation = cached_explanation
382
+ explanation_cached = cached_explanation is not None
383
+
384
+ # Generate if requested and not cached
385
+ if generate and not cached_explanation and jd_data:
386
+ try:
387
+ explanation = explainer.generate_explanation(jd_data, detail)
388
+ storage.save_explanation(jd_id, batch_id, candidate_id, explanation)
389
+ explanation_cached = False
390
+ except Exception as e:
391
+ logger.warning(f"Explanation generation failed: {e}")
392
+ explanation = None
393
+
394
+ bd_raw = detail.get("score_breakdown", "{}")
395
+ if isinstance(bd_raw, str):
396
+ try:
397
+ bd_raw = json.loads(bd_raw)
398
+ except Exception:
399
+ bd_raw = {}
400
+
401
+ flags = detail.get("flags", [])
402
+ if isinstance(flags, str):
403
+ try:
404
+ flags = json.loads(flags)
405
+ except Exception:
406
+ flags = []
407
+
408
+ def _parse_skill_list(raw):
409
+ if isinstance(raw, str):
410
+ try:
411
+ raw = json.loads(raw)
412
+ except Exception:
413
+ return []
414
+ return raw or []
415
+
416
+ matched = [SkillMatchInfo(**s) for s in _parse_skill_list(bd_raw.get("matched_skills", []))]
417
+ below = [SkillMatchInfo(**s) for s in _parse_skill_list(bd_raw.get("below_level_skills", []))]
418
+
419
+ return CandidateDetailResponse(
420
+ rank=int(detail.get("rank", 0)),
421
+ candidate_id=candidate_id,
422
+ name=detail.get("candidate_name", ""),
423
+ current_position=detail.get("current_position", ""),
424
+ years_of_experience=float(detail.get("years_of_experience", 0)),
425
+ engineer_type=detail.get("engineer_type", ""),
426
+ final_score=float(detail.get("final_score", 0)),
427
+ tier=detail.get("tier", "Partial Match"),
428
+ score_breakdown=ScoreBreakdownResponse(
429
+ skill_score=float(bd_raw.get("skill_score", 0)),
430
+ semantic_score=float(bd_raw.get("semantic_score", 0)),
431
+ position_score=float(bd_raw.get("position_score", 0)),
432
+ special_score=float(bd_raw.get("special_score", 0)),
433
+ final_score=float(bd_raw.get("final_score", 0)),
434
+ penalties=bd_raw.get("penalties", []),
435
+ ),
436
+ matched_skills=matched,
437
+ below_level_skills=below,
438
+ missing_skills=_parse_skill_list(bd_raw.get("missing_skills", [])),
439
+ flags=flags,
440
+ notice_period=detail.get("notice_period") or None,
441
+ most_recent_company=detail.get("most_recent_company") or None,
442
+ explanation=explanation,
443
+ explanation_cached=explanation_cached,
444
+ )
445
+
446
+
447
+ @router.get("/status/{match_id}", response_model=StatusResponse)
448
+ def get_status(match_id: str):
449
+ """
450
+ Status check endpoint.
451
+ Declared LAST but prefixed with literal "status/" β€” FastAPI/Starlette resolves
452
+ static-prefix routes before variable-segment routes at the same depth only when
453
+ both are GET and same length. To be safe this is also on the status_router
454
+ with its own prefix (see main.py include). Use /api/v1/status/{match_id}.
455
+ """
456
+ status = storage.get_match_status(match_id)
457
+ if not status:
458
+ raise HTTPException(404, f"Match {match_id} not found")
459
+ return StatusResponse(
460
+ match_id=match_id,
461
+ status=status.get("status", "unknown"),
462
+ progress=float(status["progress"]) if "progress" in status else None,
463
+ message=status.get("message"),
464
+ completed_at=status.get("completed_at"),
465
+ )
466
+
467
+
468
+ # Separate router for status β€” avoids /{jd_id} shadowing /status/{match_id}
469
+ status_router = APIRouter(prefix="/api/v1/status", tags=["status"])
470
+
471
+
472
+ @status_router.get("/{match_id}", response_model=StatusResponse)
473
+ def get_status_v2(match_id: str):
474
+ return get_status(match_id)
app/config.py ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic_settings import BaseSettings
2
+ from functools import lru_cache
3
+
4
+
5
+ class Settings(BaseSettings):
6
+ # Ollama cloud
7
+ ollama_host: str = "https://ollama.com"
8
+ ollama_api_key: str = ""
9
+ ollama_model: str = "gpt-oss:120b-cloud"
10
+
11
+ # Redis
12
+ redis_url: str = "redis://localhost:6379"
13
+
14
+ # CORS (comma-separated)
15
+ cors_origins: str = "http://localhost:3000"
16
+
17
+ # App
18
+ debug: bool = False
19
+ log_level: str = "INFO"
20
+
21
+ # Model
22
+ embedding_model: str = "all-MiniLM-L6-v2"
23
+
24
+ # Matching
25
+ bm25_prefilter_threshold: int = 200
26
+ bm25_top_k: int = 500
27
+
28
+ @property
29
+ def cors_origins_list(self) -> list[str]:
30
+ return [o.strip() for o in self.cors_origins.split(",") if o.strip()]
31
+
32
+ model_config = {"env_file": ".env", "case_sensitive": False, "extra": "ignore"}
33
+
34
+
35
+ @lru_cache
36
+ def get_settings() -> Settings:
37
+ return Settings()
38
+
39
+
40
+ settings = get_settings()
app/main.py ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ from contextlib import asynccontextmanager
3
+
4
+ from fastapi import FastAPI
5
+ from fastapi.middleware.cors import CORSMiddleware
6
+
7
+ from app.config import settings
8
+ from app.api.routes import health, jd, candidates, match
9
+ from app.api.routes.match import status_router
10
+ from app.pipeline import embedder
11
+
12
+ logging.basicConfig(
13
+ level=getattr(logging, settings.log_level.upper(), logging.INFO),
14
+ format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
15
+ )
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ @asynccontextmanager
20
+ async def lifespan(app: FastAPI):
21
+ # Pre-load embedding model at startup so first request is fast
22
+ logger.info("Loading embedding model...")
23
+ embedder.load_model(settings.embedding_model)
24
+ logger.info("Startup complete")
25
+ yield
26
+ logger.info("Shutdown")
27
+
28
+
29
+ app = FastAPI(
30
+ title="JD Candidate Matching API",
31
+ description="Match and rank candidates against job descriptions using hybrid scoring.",
32
+ version="1.0.0",
33
+ lifespan=lifespan,
34
+ )
35
+
36
+ app.add_middleware(
37
+ CORSMiddleware,
38
+ allow_origins=settings.cors_origins_list,
39
+ allow_credentials=True,
40
+ allow_methods=["*"],
41
+ allow_headers=["*"],
42
+ )
43
+
44
+ app.include_router(health.router)
45
+ app.include_router(jd.router)
46
+ app.include_router(candidates.router)
47
+ app.include_router(match.router)
48
+ app.include_router(status_router)
49
+
50
+
51
+ @app.get("/")
52
+ def root():
53
+ return {
54
+ "service": "JD Candidate Matching API",
55
+ "version": "1.0.0",
56
+ "docs": "/docs",
57
+ }
app/models/__init__.py ADDED
File without changes
app/models/domain.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+ from typing import Optional
3
+ from enum import Enum
4
+
5
+
6
+ class MatchTier(str, Enum):
7
+ STRONG = "Strong Match"
8
+ GOOD = "Good Match"
9
+ PARTIAL = "Partial Match"
10
+ WEAK = "Weak Match"
11
+
12
+
13
+ @dataclass
14
+ class SkillEntry:
15
+ name: str # normalized (lowercase, aliased)
16
+ raw: str # original text
17
+ level: int # 1–5
18
+ qualifier: Optional[str] = None
19
+
20
+
21
+ @dataclass
22
+ class SkillRequirement:
23
+ name: str
24
+ level: int # required proficiency 1–5
25
+ is_required: bool = True
26
+ raw: str = ""
27
+
28
+
29
+ @dataclass
30
+ class WorkExperience:
31
+ company: str
32
+ role: str
33
+ duration: str
34
+ description: str
35
+
36
+
37
+ @dataclass
38
+ class CandidateData:
39
+ id: str
40
+ name: str
41
+ current_position: str
42
+ current_title: Optional[str]
43
+ years_of_experience: float # -1.0 means unknown
44
+ engineer_type: str
45
+ gen_ai_experience: Optional[str]
46
+ parsed_skills: str
47
+ parsed_summary: str
48
+ parsed_work_experience_raw: str
49
+ parsed_work_experience_text: str # flattened plain text
50
+ work_experiences: list[WorkExperience]
51
+ extracted_skills: list[SkillEntry]
52
+ most_recent_company: Optional[str]
53
+ notice_period: Optional[str]
54
+ is_actively_looking: Optional[bool]
55
+ flags: list[str] = field(default_factory=list)
56
+ penalties: list[float] = field(default_factory=list)
57
+
58
+
59
+ @dataclass
60
+ class JDRequirements:
61
+ id: str
62
+ title: str
63
+ engineer_type: Optional[str] # None = unknown β†’ skip type filter
64
+ min_years: float # 0.0 = not specified
65
+ required_skills: list[SkillRequirement]
66
+ preferred_skills: list[SkillRequirement]
67
+ responsibilities: list[str]
68
+ raw_text: str
69
+ full_text: str
70
+
71
+
72
+ @dataclass
73
+ class SkillMatchDetail:
74
+ skill: str
75
+ candidate_level: int
76
+ required_level: int
77
+ score: float
78
+ status: str # "met", "exceeded", "below_level", "missing", "inferred"
79
+
80
+
81
+ @dataclass
82
+ class ScoreBreakdown:
83
+ skill_score: float
84
+ semantic_score: float
85
+ position_score: float
86
+ special_score: float
87
+ final_score: float
88
+ penalties_applied: list[dict]
89
+ matched_skills: list[SkillMatchDetail]
90
+ below_level_skills: list[SkillMatchDetail]
91
+ missing_skills: list[str]
92
+
93
+
94
+ @dataclass
95
+ class MatchResult:
96
+ candidate_id: str
97
+ candidate_name: str
98
+ current_position: str
99
+ years_of_experience: float
100
+ engineer_type: str
101
+ final_score: float
102
+ tier: MatchTier
103
+ score_breakdown: ScoreBreakdown
104
+ flags: list[str]
105
+ notice_period: Optional[str]
106
+ most_recent_company: Optional[str]
107
+ rank: int = 0
app/models/schemas.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic import BaseModel, Field
2
+ from typing import Optional, Any
3
+
4
+
5
+ # ─── JD Schemas ──────────────────────────────────────────────────────────────
6
+
7
+ class JDUploadResponse(BaseModel):
8
+ jd_id: str
9
+ title: str
10
+ engineer_type: Optional[str]
11
+ min_years: float
12
+ required_skills_count: int
13
+ preferred_skills_count: int
14
+ uploaded_at: str
15
+
16
+
17
+ class JDListItem(BaseModel):
18
+ jd_id: str
19
+ title: str
20
+ engineer_type: Optional[str]
21
+ min_years: float
22
+ required_skills_count: int
23
+ uploaded_at: str
24
+
25
+
26
+ class JDDetailResponse(JDListItem):
27
+ required_skills: list[dict]
28
+ preferred_skills: list[dict]
29
+ responsibilities: list[str]
30
+
31
+
32
+ # ─── Candidate Schemas ────────────────────────────────────────────────────────
33
+
34
+ class CandidateBatchResponse(BaseModel):
35
+ batch_id: str
36
+ total_count: int
37
+ engineer_type_distribution: dict[str, int]
38
+ uploaded_at: str
39
+
40
+
41
+ class BatchListItem(BaseModel):
42
+ batch_id: str
43
+ total_count: int
44
+ engineer_type_distribution: dict[str, int]
45
+ uploaded_at: str
46
+
47
+
48
+ # ─── Match Schemas ────────────────────────────────────────────────────────────
49
+
50
+ class MatchTriggerRequest(BaseModel):
51
+ batch_id: str
52
+ top_n: int = Field(default=10, ge=1, le=50)
53
+
54
+
55
+ class MatchTriggerResponse(BaseModel):
56
+ match_id: str
57
+ status: str # "processing" | "completed"
58
+ total_candidates: int
59
+ filtered_candidates: int
60
+ results: Optional[list["CandidateMatchResponse"]] = None
61
+
62
+
63
+ class SkillMatchInfo(BaseModel):
64
+ skill: str
65
+ candidate_level: int
66
+ required_level: int
67
+ score: float
68
+ status: str
69
+
70
+
71
+ class ScoreBreakdownResponse(BaseModel):
72
+ skill_score: float
73
+ semantic_score: float
74
+ position_score: float
75
+ special_score: float
76
+ final_score: float
77
+ penalties: list[dict[str, Any]]
78
+
79
+
80
+ class CandidateMatchResponse(BaseModel):
81
+ rank: int
82
+ candidate_id: str
83
+ name: str
84
+ current_position: str
85
+ years_of_experience: float
86
+ engineer_type: str
87
+ final_score: float
88
+ tier: str
89
+ score_breakdown: ScoreBreakdownResponse
90
+ matched_skills: list[SkillMatchInfo]
91
+ below_level_skills: list[SkillMatchInfo]
92
+ missing_skills: list[str]
93
+ flags: list[str]
94
+ notice_period: Optional[str]
95
+ most_recent_company: Optional[str]
96
+
97
+
98
+ class MatchResultsResponse(BaseModel):
99
+ jd_id: str
100
+ jd_title: str
101
+ batch_id: str
102
+ top_n: int
103
+ total_matched: int
104
+ candidates: list[CandidateMatchResponse]
105
+
106
+
107
+ class CandidateDetailResponse(CandidateMatchResponse):
108
+ explanation: Optional[str] = None
109
+ explanation_cached: bool = False
110
+
111
+
112
+ class StatusResponse(BaseModel):
113
+ match_id: str
114
+ status: str # "pending" | "processing" | "completed" | "failed"
115
+ progress: Optional[float] = None
116
+ message: Optional[str] = None
117
+ completed_at: Optional[str] = None
118
+
119
+
120
+ class ErrorResponse(BaseModel):
121
+ detail: str
122
+ code: Optional[str] = None
123
+
124
+
125
+ # Update forward refs
126
+ MatchTriggerResponse.model_rebuild()
app/pipeline/__init__.py ADDED
File without changes
app/pipeline/bm25_filter.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ BM25 pre-filter for large candidate pools.
3
+
4
+ Why BM25:
5
+ - O(N) scoring, no model inference needed
6
+ - Handles keyword overlap (tech terms are rare β†’ get high IDF weight)
7
+ - Cuts 100k β†’ 500 candidates in ~5 seconds on CPU
8
+
9
+ When it activates:
10
+ - Only when candidate count > BM25_PREFILTER_THRESHOLD (default 200)
11
+ - Below threshold, overhead exceeds benefit
12
+
13
+ Limitation:
14
+ - Misses semantic relationships ("microservices" β‰  "distributed systems" to BM25)
15
+ - That's fine β€” BM25 is the rough cut, embedding comparison catches the rest
16
+ """
17
+ import re
18
+ import logging
19
+ from typing import Optional
20
+
21
+ import numpy as np
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+
26
+ def _tokenize(text: str) -> list[str]:
27
+ """
28
+ Tokenize for BM25.
29
+
30
+ Preserves tech terms: "c++", "node.js", "ci/cd", ".net".
31
+ Removes very short tokens that carry no signal.
32
+ """
33
+ text = text.lower()
34
+ # Keep alphanumeric + .-+/#
35
+ text = re.sub(r"[^\w\s.+#/\-]", " ", text)
36
+ tokens = text.split()
37
+ # Filter: keep tokens with >1 char, or known short terms
38
+ return [t for t in tokens if len(t) > 1 or t in ("c", "r", "go")]
39
+
40
+
41
+ def bm25_prefilter(
42
+ jd_text: str,
43
+ candidates: list, # list[CandidateData]
44
+ top_k: int = 500,
45
+ ) -> list:
46
+ """
47
+ Score all candidates with BM25 against JD text.
48
+ Returns top_k candidates sorted by BM25 score descending.
49
+
50
+ If len(candidates) <= top_k, returns unchanged (no overhead).
51
+ """
52
+ if len(candidates) <= top_k:
53
+ return candidates
54
+
55
+ try:
56
+ from rank_bm25 import BM25Okapi
57
+ except ImportError:
58
+ logger.warning("rank-bm25 not installed, skipping BM25 pre-filter")
59
+ return candidates
60
+
61
+ logger.info(f"BM25 pre-filter: {len(candidates)} β†’ top {top_k}")
62
+
63
+ # Build corpus: combine skills + summary + work experience text
64
+ corpus: list[list[str]] = []
65
+ for c in candidates:
66
+ text = " ".join([
67
+ c.parsed_skills,
68
+ c.parsed_summary,
69
+ c.parsed_work_experience_text,
70
+ c.current_position,
71
+ ])
72
+ corpus.append(_tokenize(text))
73
+
74
+ bm25 = BM25Okapi(corpus)
75
+ query_tokens = _tokenize(jd_text)
76
+ scores = bm25.get_scores(query_tokens) # shape: (N,)
77
+
78
+ top_indices = np.argsort(scores)[::-1][:top_k]
79
+ filtered = [candidates[i] for i in top_indices]
80
+
81
+ logger.info(f"BM25 complete: top scores {scores[top_indices[:3]]}")
82
+ return filtered
app/pipeline/embedder.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Local embedding generation using sentence-transformers.
3
+
4
+ Zero API cost. Model loaded once at startup (singleton via lifespan).
5
+ Batch processing for efficiency at 100k scale.
6
+
7
+ Model: all-MiniLM-L6-v2
8
+ - 384 dimensions
9
+ - ~80 MB on disk
10
+ - ~100ms per batch of 64 on CPU
11
+ - Embeddings are L2-normalized β†’ cosine similarity = dot product
12
+ """
13
+ import logging
14
+ import threading
15
+ from typing import Optional
16
+
17
+ import numpy as np
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+ _model = None
22
+ _model_lock = threading.Lock()
23
+
24
+
25
+ def load_model(model_name: str = "all-MiniLM-L6-v2") -> None:
26
+ """Load (or warm up) the embedding model. Call once at startup."""
27
+ global _model
28
+ with _model_lock:
29
+ if _model is not None:
30
+ return
31
+ logger.info(f"Loading embedding model: {model_name}")
32
+ try:
33
+ from sentence_transformers import SentenceTransformer
34
+ _model = SentenceTransformer(model_name)
35
+ logger.info("Embedding model ready")
36
+ except Exception as e:
37
+ logger.error(f"Failed to load embedding model: {e}")
38
+ raise
39
+
40
+
41
+ def _get_model():
42
+ global _model
43
+ if _model is None:
44
+ load_model()
45
+ return _model
46
+
47
+
48
+ def embed_texts(texts: list[str], batch_size: int = 64) -> np.ndarray:
49
+ """
50
+ Embed a list of texts.
51
+
52
+ Returns ndarray of shape (N, 384), L2-normalized.
53
+ Cosine similarity between two rows = their dot product.
54
+ """
55
+ model = _get_model()
56
+ if not texts:
57
+ return np.empty((0, 384), dtype=np.float32)
58
+
59
+ embeddings = model.encode(
60
+ texts,
61
+ normalize_embeddings=True,
62
+ batch_size=batch_size,
63
+ show_progress_bar=False,
64
+ )
65
+ return embeddings.astype(np.float32)
66
+
67
+
68
+ def embed_single(text: str) -> np.ndarray:
69
+ """Embed one text. Returns shape (384,)."""
70
+ return embed_texts([text])[0]
71
+
72
+
73
+ def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
74
+ """
75
+ Cosine similarity between two L2-normalized vectors.
76
+ Since both are normalized, dot product == cosine similarity.
77
+ Returns value in [-1, 1].
78
+ """
79
+ return float(np.dot(a, b))
80
+
81
+
82
+ def normalize_similarity(sim: float) -> float:
83
+ """Map [-1, 1] similarity to [0, 1] score."""
84
+ return (sim + 1.0) / 2.0
85
+
86
+
87
+ def batch_cosine_similarity(
88
+ query: np.ndarray, corpus: np.ndarray
89
+ ) -> np.ndarray:
90
+ """
91
+ Compute cosine similarity of query against entire corpus in one shot.
92
+
93
+ query: shape (D,)
94
+ corpus: shape (N, D)
95
+ Returns: shape (N,) with scores in [-1, 1]
96
+
97
+ At 100k candidates (D=384): ~10ms on CPU.
98
+ """
99
+ return corpus @ query
app/pipeline/jd_parser.py ADDED
@@ -0,0 +1,308 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Extract structured requirements from any JD text (PDF-extracted).
3
+
4
+ Works generically β€” not hardcoded to AI Engineer / Backend Engineer.
5
+ Extracts:
6
+ - Role title
7
+ - Engineer type (inferred from title + content)
8
+ - Minimum years of experience
9
+ - Required skills (from Requirements / Core Requirements section)
10
+ - Preferred skills (from Preferred Qualifications section)
11
+ - Key responsibilities
12
+ """
13
+ import re
14
+ import uuid
15
+ from typing import Optional
16
+
17
+ from app.models.domain import JDRequirements, SkillRequirement
18
+ from app.pipeline.skill_parser import (
19
+ parse_skill_token,
20
+ normalize_skill_name,
21
+ SKILL_ALIASES,
22
+ )
23
+
24
+ # ─── Engineer type inference ──────────────────────────────────────────────────
25
+
26
+ _TYPE_PATTERNS: list[tuple[list[str], str]] = [
27
+ (["ai engineer", "ml engineer", "machine learning engineer", "nlp engineer",
28
+ "data scientist", "applied scientist"], "AI Engineer"),
29
+ (["backend engineer", "back-end engineer", "back end engineer",
30
+ "server side engineer", "api engineer", "node.js developer",
31
+ "python developer", "java developer"], "Backend Engineer"),
32
+ (["frontend engineer", "front-end engineer", "front end engineer",
33
+ "ui engineer", "react developer", "angular developer",
34
+ "vue developer", "web developer"], "Frontend Engineer"),
35
+ (["fullstack", "full stack", "full-stack"], "Full Stack Engineer"),
36
+ (["data engineer", "etl developer", "pipeline engineer"], "Data Engineer"),
37
+ (["devops engineer", "sre", "platform engineer", "cloud engineer",
38
+ "infrastructure engineer"], "DevOps Engineer"),
39
+ (["mobile engineer", "ios developer", "android developer",
40
+ "react native developer", "flutter developer"], "Mobile Engineer"),
41
+ (["software engineer", "software developer", "sde", "swe"], "Software Engineer"),
42
+ ]
43
+
44
+
45
+ def infer_engineer_type(title: str, text: str) -> Optional[str]:
46
+ combined = (title + " " + text[:500]).lower()
47
+ for keywords, eng_type in _TYPE_PATTERNS:
48
+ if any(kw in combined for kw in keywords):
49
+ return eng_type
50
+ return None
51
+
52
+
53
+ # ─── Years extraction ─────────────────────────────────────────────────────────
54
+
55
+ _YEARS_PATTERNS = [
56
+ r"minimum\s+(\d+(?:\.\d+)?)\s*\+?\s*years?",
57
+ r"at\s+least\s+(\d+(?:\.\d+)?)\s*\+?\s*years?",
58
+ r"(\d+)\+?\s*years?\s+of\s+(?:relevant\s+)?experience",
59
+ r"(\d+)\+?\s*years?\s+(?:working\s+)?(?:in|with|of)",
60
+ r"experience[:\s]+(\d+)\+?\s*years?",
61
+ r"(\d+)\s*-\s*\d+\s*years?\s+of\s+experience",
62
+ ]
63
+
64
+
65
+ def extract_min_years(text: str) -> float:
66
+ lower = text.lower()
67
+ for pattern in _YEARS_PATTERNS:
68
+ m = re.search(pattern, lower)
69
+ if m:
70
+ try:
71
+ return float(m.group(1))
72
+ except ValueError:
73
+ pass
74
+ return 0.0
75
+
76
+
77
+ # ─── Title extraction ─────────────────────────────────────────────────────────
78
+
79
+ def extract_title(text: str) -> str:
80
+ lines = [l.strip() for l in text.split("\n") if l.strip()]
81
+ # First non-empty line is usually the title
82
+ if lines:
83
+ # Take up to first 80 chars, strip markdown artifacts
84
+ candidate = re.sub(r"[#*_]", "", lines[0]).strip()
85
+ if 3 < len(candidate) < 120:
86
+ return candidate
87
+ return "Unknown Role"
88
+
89
+
90
+ # ─── Tech skills vocabulary ───────────────────────────────────────────────────
91
+
92
+ # Comprehensive vocabulary for matching skills mentioned in JD text
93
+ TECH_VOCAB: set[str] = {
94
+ # Languages
95
+ "python", "java", "javascript", "typescript", "go", "golang", "rust",
96
+ "c++", "c#", "ruby", "kotlin", "swift", "scala", "r", "php",
97
+ # Python frameworks
98
+ "fastapi", "django", "flask", "celery", "sqlalchemy",
99
+ # JS frameworks
100
+ "react", "angular", "vue", "nextjs", "express", "nodejs",
101
+ # AI/ML
102
+ "rag", "llm", "embeddings", "vector databases", "langchain", "langgraph",
103
+ "tensorflow", "pytorch", "scikit-learn", "huggingface", "transformers",
104
+ "generative ai", "nlp", "computer vision", "mlops", "fine-tuning",
105
+ "prompt engineering", "openai", "anthropic", "bedrock",
106
+ "pandas", "numpy", "matplotlib", "seaborn",
107
+ "apache spark", "airflow", "mlflow", "weights & biases",
108
+ # Databases
109
+ "postgresql", "mysql", "mongodb", "redis", "elasticsearch",
110
+ "cassandra", "dynamodb", "sqlite", "oracle", "mssql",
111
+ "qdrant", "pinecone", "weaviate", "chroma", "faiss",
112
+ # Cloud / DevOps
113
+ "aws", "gcp", "azure", "docker", "kubernetes", "terraform",
114
+ "github actions", "jenkins", "ci/cd", "lambda", "s3", "ec2",
115
+ # Messaging / async
116
+ "kafka", "rabbitmq", "celery", "sqs", "pubsub",
117
+ # APIs / patterns
118
+ "rest apis", "graphql", "grpc", "websockets", "microservices",
119
+ "serverless", "api gateway",
120
+ # Tools
121
+ "git", "jira", "confluence",
122
+ # Concepts
123
+ "system design", "distributed systems", "agile", "tdd",
124
+ "data structures", "algorithms",
125
+ # Java ecosystem
126
+ "springboot", "spring", "hibernate", "maven", "gradle",
127
+ # .NET
128
+ ".net", "asp.net",
129
+ }
130
+
131
+
132
+ def extract_skills_from_text(
133
+ text: str, is_required: bool
134
+ ) -> list[SkillRequirement]:
135
+ """
136
+ Scan text for known tech skills and extract with proficiency context.
137
+ Returns deduplicated list.
138
+ """
139
+ lower = text.lower()
140
+ found: dict[str, SkillRequirement] = {}
141
+
142
+ for skill in sorted(TECH_VOCAB, key=len, reverse=True):
143
+ if skill not in lower:
144
+ continue
145
+
146
+ # Find position and extract surrounding context (40 chars before)
147
+ idx = lower.find(skill)
148
+ context_before = lower[max(0, idx - 40): idx]
149
+ context_after = lower[idx + len(skill): idx + len(skill) + 30]
150
+ context = context_before + " " + context_after
151
+
152
+ # Determine level from context qualifiers
153
+ level = _level_from_context(context)
154
+
155
+ normalized = normalize_skill_name(skill)
156
+ if normalized not in found:
157
+ found[normalized] = SkillRequirement(
158
+ name=normalized,
159
+ level=level,
160
+ is_required=is_required,
161
+ raw=skill,
162
+ )
163
+
164
+ return list(found.values())
165
+
166
+
167
+ def _level_from_context(context: str) -> int:
168
+ """Infer required proficiency from surrounding words."""
169
+ from app.pipeline.skill_parser import _QUALIFIER_MAP, _SORTED_QUALIFIERS
170
+ lower = context.lower()
171
+ for qualifier in _SORTED_QUALIFIERS:
172
+ if qualifier in lower:
173
+ return _QUALIFIER_MAP[qualifier]
174
+ return 3 # default: intermediate
175
+
176
+
177
+ # ─── Section extraction ───────────────────────────────────────────────────────
178
+
179
+ _REQUIRED_HEADERS = [
180
+ "core requirements", "requirements", "required qualifications",
181
+ "must have", "what you need", "you will need", "technical requirements",
182
+ "skills required",
183
+ ]
184
+ _PREFERRED_HEADERS = [
185
+ "preferred qualifications", "preferred", "nice to have",
186
+ "bonus points", "brownie points", "good to have",
187
+ "what would be great", "plus",
188
+ ]
189
+ _RESPONSIBILITY_HEADERS = [
190
+ "responsibilities", "what you'll do", "what you will do",
191
+ "your role", "duties", "key responsibilities",
192
+ "role and responsibilities",
193
+ ]
194
+
195
+
196
+ def _extract_section(text: str, headers: list[str]) -> str:
197
+ """Return text between the first matching header and the next section break."""
198
+ lower = text.lower()
199
+ start = -1
200
+ for header in headers:
201
+ idx = lower.find(header)
202
+ if idx != -1:
203
+ start = idx
204
+ break
205
+
206
+ if start == -1:
207
+ return ""
208
+
209
+ # Find next section break (another header or double newline after 50+ chars)
210
+ end = len(text)
211
+ for other_headers in [_REQUIRED_HEADERS, _PREFERRED_HEADERS, _RESPONSIBILITY_HEADERS]:
212
+ for h in other_headers:
213
+ idx = lower.find(h, start + 20)
214
+ if idx != -1 and idx < end:
215
+ end = idx
216
+
217
+ return text[start:end]
218
+
219
+
220
+ def _parse_bullets(section_text: str) -> list[str]:
221
+ """Extract bullet points from a section."""
222
+ bullets: list[str] = []
223
+ for line in section_text.split("\n"):
224
+ line = line.strip()
225
+ # Remove bullet markers
226
+ line = re.sub(r"^[β€’Β·\-\*β—βž€β–Ίβ–Έβ–ͺ◦‣⁃]\s*", "", line).strip()
227
+ if len(line) > 5:
228
+ bullets.append(line)
229
+ return bullets
230
+
231
+
232
+ # ─── Skills from "Skills Required" comma list ─────────────────────────────────
233
+
234
+ def _parse_skills_section(text: str) -> list[str]:
235
+ """Find 'Skills Required' or similar and return comma/newline split items."""
236
+ lower = text.lower()
237
+ for header in ["skills required", "technical skills", "technologies", "key skills"]:
238
+ idx = lower.find(header)
239
+ if idx != -1:
240
+ # Grab next 500 chars
241
+ snippet = text[idx: idx + 500]
242
+ # Split by comma or newline
243
+ tokens = re.split(r"[,\n]", snippet)
244
+ # Skip the header line itself
245
+ result = []
246
+ for t in tokens[1:]:
247
+ t = t.strip().strip("β€’Β·-*●").strip()
248
+ if 2 < len(t) < 60:
249
+ result.append(t)
250
+ return result
251
+ return []
252
+
253
+
254
+ # ─── Main JD parser ───────────────────────────────────────────────────────────
255
+
256
+ def parse_jd(text: str, jd_id: Optional[str] = None) -> JDRequirements:
257
+ if jd_id is None:
258
+ jd_id = str(uuid.uuid4())[:8]
259
+
260
+ title = extract_title(text)
261
+ engineer_type = infer_engineer_type(title, text)
262
+ min_years = extract_min_years(text)
263
+
264
+ # Extract sections
265
+ req_section = _extract_section(text, _REQUIRED_HEADERS)
266
+ pref_section = _extract_section(text, _PREFERRED_HEADERS)
267
+ resp_section = _extract_section(text, _RESPONSIBILITY_HEADERS)
268
+
269
+ # Parse skills from dedicated sections using vocab matching
270
+ required_skills = extract_skills_from_text(req_section or text[:800], is_required=True)
271
+ preferred_skills = extract_skills_from_text(pref_section, is_required=False)
272
+
273
+ # Also parse comma-separated "Skills Required" section if present.
274
+ # ONLY add tokens that match known TECH_VOCAB β€” prevents job titles ("AI Engineer"),
275
+ # section headers ("Overview"), and sentence fragments ("and managing iot") from
276
+ # leaking into the required skills list.
277
+ _normalized_vocab = {normalize_skill_name(s) for s in TECH_VOCAB}
278
+ comma_skills = _parse_skills_section(text)
279
+ existing_names = {s.name for s in required_skills}
280
+ for raw_skill in comma_skills:
281
+ entry = parse_skill_token(raw_skill)
282
+ if entry.name and entry.name not in existing_names and entry.name in _normalized_vocab:
283
+ required_skills.append(SkillRequirement(
284
+ name=entry.name,
285
+ level=entry.level,
286
+ is_required=True,
287
+ raw=raw_skill,
288
+ ))
289
+ existing_names.add(entry.name)
290
+
291
+ # Responsibilities as plain bullets
292
+ responsibilities = _parse_bullets(resp_section)[:8]
293
+
294
+ # Fallback: if we extracted 0 required skills, scan full text
295
+ if not required_skills:
296
+ required_skills = extract_skills_from_text(text, is_required=True)
297
+
298
+ return JDRequirements(
299
+ id=jd_id,
300
+ title=title,
301
+ engineer_type=engineer_type,
302
+ min_years=min_years,
303
+ required_skills=required_skills,
304
+ preferred_skills=preferred_skills,
305
+ responsibilities=responsibilities,
306
+ raw_text=text[:200],
307
+ full_text=text,
308
+ )
app/pipeline/matcher.py ADDED
@@ -0,0 +1,543 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Core matching pipeline.
3
+
4
+ Scoring weights:
5
+ Skill matching 35% β€” proficiency-aware, no LLM
6
+ Semantic (work) 40% β€” local embeddings (sentence-transformers)
7
+ Position align 10% β€” current role vs JD title embedding
8
+ Special 15% β€” GenAI depth (AI JDs) / stack alignment (Backend JDs)
9
+
10
+ Penalties applied multiplicatively after weighted sum:
11
+ years_slightly_below: Γ—0.85 (within 20% of threshold)
12
+ years_unknown: Γ—0.95
13
+ engineer_type_unknown: Γ—0.90
14
+ work_exp_unavailable: Γ—0.95
15
+ skills_unavailable: Γ—0.80
16
+
17
+ Scale behaviour:
18
+ ≀200 candidates : skip BM25, run full scoring
19
+ >200 candidates : BM25 pre-filter to top 500, then full scoring
20
+ """
21
+ import logging
22
+ from typing import Optional
23
+
24
+ import numpy as np
25
+
26
+ from app.models.domain import (
27
+ CandidateData, JDRequirements, MatchResult, MatchTier,
28
+ ScoreBreakdown, SkillMatchDetail,
29
+ )
30
+ from app.pipeline import skill_parser, embedder as emb, bm25_filter
31
+ from app.config import settings
32
+
33
+ logger = logging.getLogger(__name__)
34
+
35
+ # ─── Tier thresholds ──────────────────────────────────────────────────────────
36
+
37
+ TIER_THRESHOLDS = [
38
+ (0.80, MatchTier.STRONG),
39
+ (0.65, MatchTier.GOOD),
40
+ (0.45, MatchTier.PARTIAL),
41
+ (0.00, MatchTier.WEAK),
42
+ ]
43
+
44
+
45
+ def _get_tier(score: float) -> MatchTier:
46
+ for threshold, tier in TIER_THRESHOLDS:
47
+ if score >= threshold:
48
+ return tier
49
+ return MatchTier.WEAK
50
+
51
+
52
+ # ─── Penalty constants ────────────────────────────────────────────────────────
53
+
54
+ PENALTY_YEARS_BELOW = 0.15
55
+ PENALTY_YEARS_UNKNOWN = 0.05
56
+ PENALTY_TYPE_UNKNOWN = 0.10
57
+ PENALTY_NO_WORK_EXP = 0.05
58
+ PENALTY_NO_SKILLS = 0.20
59
+
60
+
61
+ # ─── GenAI score lookup ───────────────────────────────────────────────────────
62
+
63
+ _GEN_AI_SCORE_MAP: dict[str, float] = {
64
+ "2+ years": 1.0,
65
+ "1.5 years - 2 years": 0.85,
66
+ "1 to 1.5 years": 0.75,
67
+ "6 months - 1 year": 0.60,
68
+ "only internship/project experience": 0.40,
69
+ "less than 6 months": 0.35,
70
+ "i do not have gen ai experience": 0.10,
71
+ }
72
+
73
+ _GEN_AI_INFERENCE_KEYWORDS = {
74
+ "rag", "llm", "vector", "embedding", "langchain", "langgraph",
75
+ "openai", "anthropic", "generative", "gen ai", "genai",
76
+ "huggingface", "transformer", "gpt", "claude", "gemini",
77
+ "fine-tun", "prompt engineer",
78
+ }
79
+
80
+
81
+ def _score_gen_ai(candidate: CandidateData) -> float:
82
+ """
83
+ Score GenAI experience with claim cross-validation.
84
+
85
+ Problem: gen_ai_experience is self-reported. Candidates claim "2+ years" with
86
+ zero LLM/RAG evidence in their actual work history. We now cross-validate
87
+ the claim against work history evidence before trusting it.
88
+
89
+ Evidence source: ONLY parsed_work_experience_text (not summary β€” summary is
90
+ also a self-authored marketing document, same credibility problem).
91
+ """
92
+ # Step 1: count GenAI keyword hits in WORK HISTORY only (not summary)
93
+ work_text = candidate.parsed_work_experience_text.lower()
94
+ work_evidence = sum(1 for kw in _GEN_AI_INFERENCE_KEYWORDS if kw in work_text)
95
+
96
+ # Map work evidence hits β†’ inferred score
97
+ if work_evidence >= 4:
98
+ inferred = 0.70
99
+ elif work_evidence >= 2:
100
+ inferred = 0.50
101
+ elif work_evidence >= 1:
102
+ inferred = 0.30
103
+ else:
104
+ inferred = 0.10
105
+
106
+ # Step 2: look up self-reported claim
107
+ raw = candidate.gen_ai_experience
108
+ claimed = None
109
+ if raw and raw.lower() not in ("none", "nan", ""):
110
+ key = raw.strip().lower()
111
+ if key in _GEN_AI_SCORE_MAP:
112
+ claimed = _GEN_AI_SCORE_MAP[key]
113
+ else:
114
+ for k, v in _GEN_AI_SCORE_MAP.items():
115
+ if k in key or key in k:
116
+ claimed = v
117
+ break
118
+
119
+ # Step 3: cross-validate claim vs evidence
120
+ if claimed is not None:
121
+ if work_evidence == 0:
122
+ # Claim exists but ZERO work history evidence β†’ claim is unverifiable
123
+ # Use skills text as a weak corroboration signal
124
+ skill_evidence = sum(
125
+ 1 for kw in _GEN_AI_INFERENCE_KEYWORDS
126
+ if kw in candidate.parsed_skills.lower()
127
+ )
128
+ skill_inferred = 0.40 if skill_evidence >= 3 else (0.25 if skill_evidence >= 1 else 0.10)
129
+ return skill_inferred # ignore the claim entirely
130
+ elif work_evidence >= 3:
131
+ # Strong work evidence β†’ trust claim fully
132
+ return claimed
133
+ else:
134
+ # Partial evidence β†’ blend: 50% claim, 50% evidence
135
+ return 0.50 * claimed + 0.50 * inferred
136
+
137
+ # No claim field β†’ pure inference (skills + work text)
138
+ combined = work_text + " " + candidate.parsed_skills.lower()
139
+ all_matches = sum(1 for kw in _GEN_AI_INFERENCE_KEYWORDS if kw in combined)
140
+ if all_matches >= 4:
141
+ return 0.70
142
+ if all_matches >= 2:
143
+ return 0.50
144
+ if all_matches >= 1:
145
+ return 0.30
146
+ return 0.10
147
+
148
+
149
+ # ─── Backend stack score ──────────────────────────────────────────────────────
150
+
151
+ _BACKEND_CORE_SKILLS = {"nodejs", "node.js", "express", "expressjs"}
152
+ _BACKEND_CLOUD = {"aws", "gcp", "azure"}
153
+ _BACKEND_DB = {"postgresql", "mongodb", "mysql", "dynamodb", "redis"}
154
+ _BACKEND_ARCH = {"microservices", "serverless", "rest apis", "api"}
155
+
156
+
157
+ def _score_backend_stack(candidate: CandidateData) -> float:
158
+ """Score stack alignment for backend JD."""
159
+ skill_names = {s.name for s in candidate.extracted_skills}
160
+ combined = candidate.parsed_skills.lower() + " " + candidate.parsed_work_experience_text.lower()
161
+
162
+ node_score = 1.0 if (skill_names & _BACKEND_CORE_SKILLS or "node" in combined) else 0.3
163
+ cloud_score = 0.8 if (skill_names & _BACKEND_CLOUD) else (0.4 if "cloud" in combined else 0.1)
164
+ db_score = 0.8 if (skill_names & _BACKEND_DB) else 0.2
165
+ arch_score = 0.7 if (skill_names & _BACKEND_ARCH or "microservice" in combined) else 0.3
166
+
167
+ return 0.4 * node_score + 0.2 * cloud_score + 0.2 * db_score + 0.2 * arch_score
168
+
169
+
170
+ # ─── Skill scoring ────────────────────────────────────────────────────────────
171
+
172
+ def _score_skills(
173
+ jd: JDRequirements,
174
+ candidate: CandidateData,
175
+ ) -> tuple[float, list[SkillMatchDetail], list[SkillMatchDetail], list[str]]:
176
+ """
177
+ Returns (skill_score, matched, below_level, missing_skill_names).
178
+
179
+ Core logic:
180
+ candidate_level >= jd_level β†’ 1.0
181
+ candidate_level == jd_level - 1 β†’ 0.6
182
+ candidate_level == jd_level - 2 β†’ 0.2
183
+ candidate_level < jd_level - 2 β†’ 0.05
184
+ skill absent from candidate β†’ 0.0
185
+
186
+ Required skills weighted 3Γ—, preferred 1Γ—.
187
+ """
188
+ if not candidate.extracted_skills:
189
+ return 0.5, [], [], []
190
+
191
+ # Map normalized name β†’ SkillEntry for fast lookup
192
+ cand_map: dict[str, "SkillEntry"] = {s.name: s for s in candidate.extracted_skills}
193
+ # Also index raw lowercased names for fuzzy matching
194
+ cand_raw_map: dict[str, "SkillEntry"] = {}
195
+ for s in candidate.extracted_skills:
196
+ cand_raw_map[s.raw.lower()] = s
197
+
198
+ # Required skills weight 3Γ—, preferred 1Γ—.
199
+ # Deduplicate across lists β€” same skill in both required+preferred would score twice.
200
+ # Keep the higher weight (required) when there's a conflict.
201
+ _seen_jd: dict[str, float] = {}
202
+ for s in jd.required_skills:
203
+ _seen_jd[s.name] = max(_seen_jd.get(s.name, 0.0), 3.0)
204
+ for s in jd.preferred_skills:
205
+ if s.name not in _seen_jd:
206
+ _seen_jd[s.name] = 1.0
207
+
208
+ # Rebuild lookup: name β†’ SkillRequirement for the deduped set
209
+ _jd_skill_map = {s.name: s for s in jd.required_skills}
210
+ _jd_skill_map.update({s.name: s for s in jd.preferred_skills})
211
+
212
+ all_jd_skills = [
213
+ (_jd_skill_map[name], weight) for name, weight in _seen_jd.items()
214
+ ]
215
+
216
+ if not all_jd_skills:
217
+ # No JD skills extracted β†’ use semantic only
218
+ return 0.5, [], [], []
219
+
220
+ total_weight = 0.0
221
+ weighted_score = 0.0
222
+ matched: list[SkillMatchDetail] = []
223
+ below_level: list[SkillMatchDetail] = []
224
+ missing: list[str] = []
225
+
226
+ for jd_skill, weight in all_jd_skills:
227
+ jd_name = jd_skill.name
228
+ jd_level = jd_skill.level
229
+
230
+ # Find candidate skill (normalized name)
231
+ cand_skill = cand_map.get(jd_name)
232
+
233
+ # Try partial match on raw name if exact fails
234
+ if cand_skill is None:
235
+ for raw, cs in cand_raw_map.items():
236
+ if jd_name in raw or raw in jd_name:
237
+ cand_skill = cs
238
+ break
239
+
240
+ if cand_skill is None:
241
+ # Check if skill appears in work experience text (inferred)
242
+ combined_lower = (
243
+ candidate.parsed_work_experience_text.lower() + " " +
244
+ candidate.parsed_skills.lower()
245
+ )
246
+ if jd_name in combined_lower or jd_skill.raw.lower() in combined_lower:
247
+ # Inferred presence β€” give partial credit at default level
248
+ level_diff = max(0, jd_level - 3)
249
+ infer_score = max(0.0, 0.5 - 0.15 * level_diff)
250
+ weighted_score += infer_score * weight
251
+ total_weight += weight
252
+ matched.append(SkillMatchDetail(
253
+ skill=jd_name, candidate_level=3,
254
+ required_level=jd_level, score=infer_score,
255
+ status="inferred",
256
+ ))
257
+ else:
258
+ missing.append(jd_name)
259
+ total_weight += weight
260
+ else:
261
+ c_level = cand_skill.level
262
+ level_diff = jd_level - c_level
263
+
264
+ if level_diff <= 0:
265
+ per_score = 1.0
266
+ status = "exceeded" if level_diff < 0 else "met"
267
+ elif level_diff == 1:
268
+ per_score = 0.60
269
+ status = "below_level"
270
+ elif level_diff == 2:
271
+ per_score = 0.20
272
+ status = "below_level"
273
+ else:
274
+ per_score = 0.05
275
+ status = "below_level"
276
+
277
+ weighted_score += per_score * weight
278
+ total_weight += weight
279
+
280
+ detail = SkillMatchDetail(
281
+ skill=jd_name,
282
+ candidate_level=c_level,
283
+ required_level=jd_level,
284
+ score=per_score,
285
+ status=status,
286
+ )
287
+ if per_score >= 0.60:
288
+ matched.append(detail)
289
+ else:
290
+ below_level.append(detail)
291
+
292
+ skill_score = weighted_score / total_weight if total_weight > 0 else 0.5
293
+ return skill_score, matched, below_level, missing
294
+
295
+
296
+ # ─── Semantic scoring ─────────────────────────────────────────────────────────
297
+
298
+ def _score_semantic(
299
+ jd_embedding: np.ndarray,
300
+ candidate: CandidateData,
301
+ candidate_embedding: np.ndarray,
302
+ ) -> float:
303
+ """Cosine similarity of JD vs candidate work+summary, normalized to [0,1]."""
304
+ sim = emb.cosine_similarity(jd_embedding, candidate_embedding)
305
+ return emb.normalize_similarity(sim)
306
+
307
+
308
+ def _score_position(
309
+ jd_title_embedding: np.ndarray,
310
+ candidate: CandidateData,
311
+ ) -> float:
312
+ """Embed current_position + current_title, compare with JD title."""
313
+ pos_parts = [p for p in [candidate.current_position, candidate.current_title] if p]
314
+ if not pos_parts:
315
+ return 0.5
316
+ pos_text = " ".join(pos_parts)
317
+ pos_emb = emb.embed_single(pos_text)
318
+ sim = emb.cosine_similarity(jd_title_embedding, pos_emb)
319
+ return emb.normalize_similarity(sim)
320
+
321
+
322
+ # ─── Filters ──────────────────────────────────────────────────────────────────
323
+
324
+ def _filter_by_type(
325
+ jd: JDRequirements,
326
+ candidates: list[CandidateData],
327
+ ) -> list[CandidateData]:
328
+ """Hard filter: exclude wrong engineer type. Include unknowns with flag."""
329
+ if not jd.engineer_type:
330
+ return candidates # No type info β€” keep all
331
+
332
+ kept: list[CandidateData] = []
333
+ for c in candidates:
334
+ if c.engineer_type == jd.engineer_type:
335
+ kept.append(c)
336
+ elif "engineer_type_unknown" in c.flags or c.engineer_type == "Unknown":
337
+ if "engineer_type_unknown" not in c.flags:
338
+ c.flags.append("engineer_type_unknown")
339
+ kept.append(c)
340
+ # else: wrong type β†’ excluded silently
341
+ return kept
342
+
343
+
344
+ def _filter_by_years(
345
+ jd: JDRequirements,
346
+ candidates: list[CandidateData],
347
+ ) -> list[CandidateData]:
348
+ """
349
+ Soft filter with 20% buffer.
350
+
351
+ - Unknown (βˆ’1): include with penalty
352
+ - Within 20% below threshold: include with penalty
353
+ - Below 20% threshold: exclude
354
+ """
355
+ if jd.min_years <= 0:
356
+ return candidates
357
+
358
+ threshold = jd.min_years
359
+ buffer_min = threshold * 0.80
360
+
361
+ kept: list[CandidateData] = []
362
+ for c in candidates:
363
+ yoe = c.years_of_experience
364
+ if yoe == -1.0:
365
+ if PENALTY_YEARS_UNKNOWN not in c.penalties:
366
+ c.penalties.append(PENALTY_YEARS_UNKNOWN)
367
+ if "years_unknown" not in c.flags:
368
+ c.flags.append("years_unknown")
369
+ kept.append(c)
370
+ elif yoe >= threshold:
371
+ kept.append(c)
372
+ elif yoe >= buffer_min:
373
+ if PENALTY_YEARS_BELOW not in c.penalties:
374
+ c.penalties.append(PENALTY_YEARS_BELOW)
375
+ if "years_slightly_below" not in c.flags:
376
+ c.flags.append("years_slightly_below")
377
+ kept.append(c)
378
+ # else: too far below β†’ excluded
379
+ return kept
380
+
381
+
382
+ # ─── Weight configuration ─────────────────────────────────────────────────────
383
+
384
+ def _get_weights(jd: JDRequirements, candidate: CandidateData) -> dict[str, float]:
385
+ """Return scoring weights based on JD type and data availability."""
386
+ has_skills = bool(candidate.extracted_skills)
387
+ has_work = bool(candidate.parsed_work_experience_text.strip())
388
+
389
+ if not has_skills and not has_work:
390
+ return {"skill": 0.0, "semantic": 0.70, "position": 0.15, "special": 0.15}
391
+ if not has_work:
392
+ return {"skill": 0.60, "semantic": 0.00, "position": 0.15, "special": 0.25}
393
+ if not jd.required_skills:
394
+ # Vague JD β€” lean on semantic
395
+ return {"skill": 0.20, "semantic": 0.55, "position": 0.10, "special": 0.15}
396
+
397
+ return {"skill": 0.35, "semantic": 0.40, "position": 0.10, "special": 0.15}
398
+
399
+
400
+ # ─── Main matching function ───────────────────────────────────────────────────
401
+
402
+ def run_matching(
403
+ jd: JDRequirements,
404
+ candidates: list[CandidateData],
405
+ top_n: int = 10,
406
+ ) -> list[MatchResult]:
407
+ """
408
+ Full pipeline:
409
+ 1. Preprocess candidate skills
410
+ 2. Hard filter: engineer_type
411
+ 3. Hard filter: years_of_experience
412
+ 4. BM25 pre-filter (if > threshold)
413
+ 5. Embed JD + all candidates (batch)
414
+ 6. Score each candidate
415
+ 7. Sort + return top_n
416
+ """
417
+ logger.info(f"Starting match: jd={jd.id}, candidates={len(candidates)}, top_n={top_n}")
418
+
419
+ # ── Step 0: extract skills for each candidate ──────────────────────────
420
+ for c in candidates:
421
+ if not c.extracted_skills:
422
+ c.extracted_skills = skill_parser.parse_skills_string(c.parsed_skills)
423
+ c.extracted_skills = skill_parser.upgrade_levels_from_work_exp(
424
+ c.extracted_skills, c.work_experiences
425
+ )
426
+ if not c.extracted_skills and "skills_unavailable" not in c.flags:
427
+ c.flags.append("skills_unavailable")
428
+ c.penalties.append(PENALTY_NO_SKILLS)
429
+ if not c.parsed_work_experience_text.strip() and "work_exp_unavailable" not in c.flags:
430
+ c.flags.append("work_exp_unavailable")
431
+ c.penalties.append(PENALTY_NO_WORK_EXP)
432
+
433
+ # ── Step 1: hard filters ────────────────────────────────────────────────
434
+ filtered = _filter_by_type(jd, candidates)
435
+ logger.info(f"After type filter: {len(filtered)}")
436
+ filtered = _filter_by_years(jd, filtered)
437
+ logger.info(f"After years filter: {len(filtered)}")
438
+
439
+ if not filtered:
440
+ logger.warning("No candidates passed filters")
441
+ return []
442
+
443
+ # ── Step 2: BM25 pre-filter ─────────────────────────────────────────────
444
+ if len(filtered) > settings.bm25_prefilter_threshold:
445
+ filtered = bm25_filter.bm25_prefilter(
446
+ jd.full_text, filtered, top_k=settings.bm25_top_k
447
+ )
448
+
449
+ # ── Step 3: compute embeddings ──────────────────────────────────────────
450
+ jd_content_text = f"{jd.title} {jd.full_text[:1500]}"
451
+ jd_content_emb = emb.embed_single(jd_content_text)
452
+ jd_title_emb = emb.embed_single(jd.title)
453
+
454
+ # Semantic embedding uses WORK HISTORY only β€” it's evidence, not marketing.
455
+ # Summary is a self-authored claim document; treating it equally to work
456
+ # history lets candidates inflate scores by writing buzzwords in their bio.
457
+ # Fallback to summary only when no work history exists (fresh grads, etc.)
458
+ candidate_texts = [
459
+ c.parsed_work_experience_text.strip() or c.parsed_summary
460
+ for c in filtered
461
+ ]
462
+ candidate_embeddings = emb.embed_texts(candidate_texts) # (N, 384)
463
+ logger.info(f"Embeddings computed for {len(filtered)} candidates")
464
+
465
+ # ── Step 4: score each candidate ────────────────────────────────────────
466
+ results: list[MatchResult] = []
467
+ is_ai_jd = jd.engineer_type == "AI Engineer"
468
+ is_backend_jd = jd.engineer_type == "Backend Engineer"
469
+
470
+ for i, candidate in enumerate(filtered):
471
+ cand_emb = candidate_embeddings[i]
472
+
473
+ # 4A: skill score
474
+ skill_score, matched, below_level, missing = _score_skills(jd, candidate)
475
+
476
+ # 4B: semantic score
477
+ semantic_score = _score_semantic(jd_content_emb, candidate, cand_emb)
478
+
479
+ # 4C: position score
480
+ position_score = _score_position(jd_title_emb, candidate)
481
+
482
+ # 4D: special score
483
+ if is_ai_jd:
484
+ special_score = _score_gen_ai(candidate)
485
+ elif is_backend_jd:
486
+ special_score = _score_backend_stack(candidate)
487
+ else:
488
+ # Unknown type β€” use position alignment again as tiebreaker
489
+ special_score = position_score
490
+
491
+ # 4E: weighted sum
492
+ weights = _get_weights(jd, candidate)
493
+ raw_score = (
494
+ weights["skill"] * skill_score
495
+ + weights["semantic"] * semantic_score
496
+ + weights["position"] * position_score
497
+ + weights["special"] * special_score
498
+ )
499
+
500
+ # 4F: apply penalties multiplicatively
501
+ final_score = raw_score
502
+ penalties_applied: list[dict] = []
503
+ for penalty in candidate.penalties:
504
+ final_score *= (1 - penalty)
505
+ penalties_applied.append({"factor": penalty, "effect": f"-{penalty*100:.0f}%"})
506
+
507
+ final_score = max(0.0, min(1.0, final_score))
508
+ tier = _get_tier(final_score)
509
+
510
+ results.append(MatchResult(
511
+ candidate_id=candidate.id,
512
+ candidate_name=candidate.name,
513
+ current_position=candidate.current_position,
514
+ years_of_experience=candidate.years_of_experience,
515
+ engineer_type=candidate.engineer_type,
516
+ final_score=final_score,
517
+ tier=tier,
518
+ score_breakdown=ScoreBreakdown(
519
+ skill_score=round(skill_score, 4),
520
+ semantic_score=round(semantic_score, 4),
521
+ position_score=round(position_score, 4),
522
+ special_score=round(special_score, 4),
523
+ final_score=round(final_score, 4),
524
+ penalties_applied=penalties_applied,
525
+ matched_skills=matched,
526
+ below_level_skills=below_level,
527
+ missing_skills=missing,
528
+ ),
529
+ flags=list(candidate.flags),
530
+ notice_period=candidate.notice_period,
531
+ most_recent_company=candidate.most_recent_company,
532
+ ))
533
+
534
+ # ── Step 5: sort and assign ranks ───────────────────────────────────────
535
+ results.sort(key=lambda r: r.final_score, reverse=True)
536
+ for rank, result in enumerate(results[:top_n], start=1):
537
+ result.rank = rank
538
+
539
+ logger.info(
540
+ f"Matching complete. Top scores: "
541
+ f"{[round(r.final_score, 3) for r in results[:5]]}"
542
+ )
543
+ return results[:top_n]
app/pipeline/skill_parser.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Proficiency-aware skill parsing.
3
+
4
+ Solves: "Core Java" (level 4) β‰  "Basic Java" (level 2) β‰  "Java" (level 3 default).
5
+
6
+ Each skill token is parsed into (normalized_name, level 1-5, qualifier_found).
7
+ Level scale:
8
+ 5 = Expert/Mastery
9
+ 4 = Core/Advanced/Strong/Deep
10
+ 3 = Intermediate/Proficient (DEFAULT for unqualified)
11
+ 2 = Basic/Beginner/Exposure
12
+ 1 = Theoretical/Academic/Awareness
13
+ """
14
+ import re
15
+ from app.models.domain import SkillEntry
16
+
17
+ # ─── Proficiency map ──────────────────────────────────────────────────────────
18
+
19
+ _QUALIFIER_LEVELS: list[tuple[int, list[str]]] = [
20
+ (5, ["expert", "mastery", "mastered", "guru", "specialist"]),
21
+ (4, [
22
+ "core", "advanced", "strong", "deep", "extensive", "senior",
23
+ "production", "in-depth", "high proficiency", "highly proficient",
24
+ ]),
25
+ (3, [
26
+ "proficient", "intermediate", "working knowledge", "solid",
27
+ "good", "hands-on", "hands on", "experienced", "skilled",
28
+ "comfortable", "competent",
29
+ ]),
30
+ (2, [
31
+ "basic", "basics", "beginner", "elementary", "familiar",
32
+ "exposure", "learning", "introductory", "intro",
33
+ "fundamental", "foundational",
34
+ ]),
35
+ (1, [
36
+ "theoretical", "academic", "theory", "awareness",
37
+ "conceptual", "introductory knowledge",
38
+ ]),
39
+ ]
40
+
41
+ # Build flat lookup: qualifier_text β†’ level (longer phrases first)
42
+ _QUALIFIER_MAP: dict[str, int] = {}
43
+ for _lvl, _quals in _QUALIFIER_LEVELS:
44
+ for _q in _quals:
45
+ _QUALIFIER_MAP[_q] = _lvl
46
+
47
+ _SORTED_QUALIFIERS = sorted(_QUALIFIER_MAP.keys(), key=len, reverse=True)
48
+
49
+ # ─── Skill alias map ──────────────────────────────────────────────────────────
50
+
51
+ SKILL_ALIASES: dict[str, str] = {
52
+ # JavaScript ecosystem
53
+ "react.js": "react", "reactjs": "react", "react js": "react",
54
+ "node.js": "nodejs", "node js": "nodejs",
55
+ "vue.js": "vue", "vuejs": "vue",
56
+ "express.js": "express", "expressjs": "express",
57
+ "next.js": "nextjs", "next js": "nextjs",
58
+ "angular.js": "angularjs",
59
+ "typescript": "typescript", "ts": "typescript",
60
+ "javascript": "javascript", "js": "javascript",
61
+ # Python ecosystem
62
+ "fast api": "fastapi", "fast-api": "fastapi",
63
+ "django rest framework": "django rest framework", "drf": "django rest framework",
64
+ "scikit learn": "scikit-learn", "sklearn": "scikit-learn",
65
+ "sci-kit learn": "scikit-learn",
66
+ "langchain": "langchain", "lang chain": "langchain",
67
+ "langraph": "langgraph", "lang graph": "langgraph",
68
+ # AI/ML
69
+ "large language models": "llm", "large language model": "llm",
70
+ "llms": "llm", "llm": "llm",
71
+ "retrieval augmented generation": "rag",
72
+ "retrieval-augmented generation": "rag",
73
+ "vector database": "vector databases", "vector db": "vector databases",
74
+ "vector store": "vector databases",
75
+ "generative ai": "generative ai", "gen ai": "generative ai",
76
+ "genai": "generative ai", "gen-ai": "generative ai",
77
+ "tensorflow": "tensorflow", "tf": "tensorflow",
78
+ "pytorch": "pytorch", "torch": "pytorch",
79
+ "hugging face": "huggingface", "hf": "huggingface",
80
+ # Databases
81
+ "postgresql": "postgresql", "postgres": "postgresql", "postgres sql": "postgresql",
82
+ "mongodb": "mongodb", "mongo": "mongodb",
83
+ "elasticsearch": "elasticsearch", "elastic search": "elasticsearch",
84
+ # DevOps/Cloud
85
+ "kubernetes": "kubernetes", "k8s": "kubernetes",
86
+ "amazon web services": "aws", "amazon aws": "aws",
87
+ "google cloud platform": "gcp", "google cloud": "gcp",
88
+ "microsoft azure": "azure",
89
+ "github actions": "github actions", "gh actions": "github actions",
90
+ # REST
91
+ "rest api": "rest apis", "restful api": "rest apis",
92
+ "restful apis": "rest apis", "rest": "rest apis",
93
+ "rest api development": "rest apis",
94
+ # Java
95
+ "spring boot": "springboot", "spring-boot": "springboot",
96
+ "java spring": "springboot",
97
+ # Misc
98
+ "message queue": "message queues", "mq": "message queues",
99
+ "rabbitmq": "rabbitmq", "rabbit mq": "rabbitmq",
100
+ "ci/cd": "ci/cd", "cicd": "ci/cd",
101
+ "websocket": "websockets", "web socket": "websockets",
102
+ "natural language processing": "nlp",
103
+ "computer vision": "computer vision", "cv": "computer vision",
104
+ }
105
+
106
+
107
+ def normalize_skill_name(raw: str) -> str:
108
+ """Lowercase, strip parens/qualifiers, apply alias."""
109
+ s = raw.lower().strip()
110
+ # Remove parenthetical suffixes: "Python (Advanced)" β†’ "python"
111
+ s = re.sub(r"\s*\(.*?\)", "", s).strip()
112
+ # Remove colon suffixes: "SQL: Advanced Queries" β†’ "sql"
113
+ s = re.sub(r":.*$", "", s).strip()
114
+ # Apply alias
115
+ return SKILL_ALIASES.get(s, s)
116
+
117
+
118
+ def parse_skill_token(token: str) -> SkillEntry:
119
+ """
120
+ Parse one skill token from a comma-separated skills list.
121
+
122
+ Returns SkillEntry with normalized name, level 1-5, qualifier string.
123
+
124
+ Examples:
125
+ "Core Java" β†’ name="java", level=4, qualifier="core"
126
+ "Basic Python" β†’ name="python", level=2, qualifier="basic"
127
+ "Python" β†’ name="python", level=3, qualifier=None (default)
128
+ "Python (Advanced)" β†’ name="python", level=4, qualifier="advanced"
129
+ "SQL: Advanced" β†’ name="sql", level=4, qualifier="advanced"
130
+ """
131
+ token = token.strip()
132
+ lower = token.lower()
133
+
134
+ # 1. Check prefix qualifiers (longest first to avoid "basic" matching "basics")
135
+ for qualifier in _SORTED_QUALIFIERS:
136
+ if lower.startswith(qualifier + " ") or lower.startswith(qualifier + "-"):
137
+ base_raw = token[len(qualifier):].strip().lstrip("-").strip()
138
+ level = _QUALIFIER_MAP[qualifier]
139
+ return SkillEntry(
140
+ name=normalize_skill_name(base_raw),
141
+ raw=token,
142
+ level=level,
143
+ qualifier=qualifier,
144
+ )
145
+
146
+ # 2. Check parenthetical suffix: "Python (Advanced)"
147
+ m = re.match(r"^(.+?)\s*\(([^)]+)\)\s*$", lower)
148
+ if m:
149
+ base_lower, paren_lower = m.group(1).strip(), m.group(2).strip()
150
+ for qualifier in _SORTED_QUALIFIERS:
151
+ if paren_lower == qualifier or paren_lower.startswith(qualifier):
152
+ base_raw = token[: token.lower().find("(")].strip()
153
+ return SkillEntry(
154
+ name=normalize_skill_name(base_raw),
155
+ raw=token,
156
+ level=_QUALIFIER_MAP[qualifier],
157
+ qualifier=qualifier,
158
+ )
159
+
160
+ # 3. Check colon pattern: "SQL: Advanced Queries"
161
+ m = re.match(r"^(.+?)\s*:\s*(\w[\w\s]*)$", lower)
162
+ if m:
163
+ base_lower, suffix_lower = m.group(1).strip(), m.group(2).strip()
164
+ for qualifier in _SORTED_QUALIFIERS:
165
+ if suffix_lower.startswith(qualifier):
166
+ base_raw = token[: token.lower().find(":")].strip()
167
+ return SkillEntry(
168
+ name=normalize_skill_name(base_raw),
169
+ raw=token,
170
+ level=_QUALIFIER_MAP[qualifier],
171
+ qualifier=qualifier,
172
+ )
173
+
174
+ # 4. No qualifier β†’ default intermediate (3)
175
+ return SkillEntry(
176
+ name=normalize_skill_name(token),
177
+ raw=token,
178
+ level=3,
179
+ qualifier=None,
180
+ )
181
+
182
+
183
+ def parse_skills_string(skills_text: str) -> list[SkillEntry]:
184
+ """Parse comma-separated skills string into structured list."""
185
+ if not skills_text or not skills_text.strip():
186
+ return []
187
+
188
+ tokens = [t.strip() for t in skills_text.split(",") if t.strip()]
189
+ entries: list[SkillEntry] = []
190
+ seen: set[str] = set()
191
+
192
+ for token in tokens:
193
+ entry = parse_skill_token(token)
194
+ if entry.name and entry.name not in seen:
195
+ seen.add(entry.name)
196
+ entries.append(entry)
197
+
198
+ return entries
199
+
200
+
201
+ def upgrade_levels_from_work_exp(
202
+ skills: list[SkillEntry],
203
+ work_experiences: list, # list[WorkExperience]
204
+ ) -> list[SkillEntry]:
205
+ """
206
+ Upgrade unqualified skills (level=3) based on depth in work experience.
207
+
208
+ Rules:
209
+ - Skill appears in job title across any role β†’ upgrade to 4
210
+ - Skill mentioned in descriptions of 2+ roles β†’ upgrade to 4
211
+ - Skill mentioned in 1 role description β†’ keep 3
212
+ - Not found β†’ keep 3 (skill may be peripheral to their work)
213
+ """
214
+ # Build work experience corpus for fast lookup
215
+ role_corpus: list[str] = []
216
+ desc_corpus: list[str] = []
217
+
218
+ for exp in work_experiences:
219
+ if hasattr(exp, "role"):
220
+ role_corpus.append(exp.role.lower())
221
+ desc_corpus.append(exp.description.lower())
222
+
223
+ upgraded: list[SkillEntry] = []
224
+ for skill in skills:
225
+ if skill.qualifier is not None or skill.level != 3:
226
+ # Already has explicit qualifier β€” don't override
227
+ upgraded.append(skill)
228
+ continue
229
+
230
+ name = skill.name.lower()
231
+ # Also check the original raw text for matching
232
+ raw_lower = skill.raw.lower()
233
+
234
+ in_title = any(name in role or raw_lower in role for role in role_corpus)
235
+ desc_matches = sum(
236
+ 1 for desc in desc_corpus if name in desc or raw_lower in desc
237
+ )
238
+
239
+ if in_title or desc_matches >= 2:
240
+ upgraded.append(SkillEntry(
241
+ name=skill.name,
242
+ raw=skill.raw,
243
+ level=4,
244
+ qualifier="inferred_from_experience",
245
+ ))
246
+ else:
247
+ upgraded.append(skill)
248
+
249
+ return upgraded
app/services/__init__.py ADDED
File without changes
app/services/explainer.py ADDED
@@ -0,0 +1,194 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Lazy explanation generation via Ollama cloud (gpt-oss:120b-cloud).
3
+
4
+ Called only when a human explicitly requests a candidate detail view.
5
+ Result cached in Redis/memory for 48h β€” subsequent requests return instantly.
6
+
7
+ Fallback: if Ollama is unavailable, returns a template explanation.
8
+ """
9
+ import json
10
+ import logging
11
+
12
+ from ollama import Client
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+ _ollama_client = None
17
+
18
+
19
+ def _get_client():
20
+ global _ollama_client
21
+ if _ollama_client is None:
22
+ from app.config import settings
23
+ _ollama_client = Client(
24
+ host=settings.ollama_host,
25
+ headers={"Authorization": f"Bearer {settings.ollama_api_key}"},
26
+ )
27
+ return _ollama_client
28
+
29
+
30
+ def _build_prompt(jd_data: dict, candidate_detail: dict) -> str:
31
+ jd_title = jd_data.get("title", "the role")
32
+ jd_type = jd_data.get("engineer_type", "")
33
+ min_years = jd_data.get("min_years", 0)
34
+
35
+ req_skills = jd_data.get("required_skills", [])
36
+ if isinstance(req_skills, str):
37
+ req_skills = json.loads(req_skills)
38
+ req_skill_names = [s.get("name", s) if isinstance(s, dict) else s for s in req_skills[:10]]
39
+
40
+ responsibilities = jd_data.get("responsibilities", [])
41
+ if isinstance(responsibilities, str):
42
+ responsibilities = json.loads(responsibilities)
43
+
44
+ name = candidate_detail.get("candidate_name", "Candidate")
45
+ position = candidate_detail.get("current_position", "")
46
+ years = candidate_detail.get("years_of_experience", 0)
47
+ skills_text = candidate_detail.get("parsed_skills", "")[:400]
48
+ summary_text = candidate_detail.get("parsed_summary", "")[:300]
49
+ work_text = candidate_detail.get("parsed_work_experience_text", "")[:600]
50
+
51
+ breakdown = candidate_detail.get("score_breakdown", {})
52
+ if isinstance(breakdown, str):
53
+ breakdown = json.loads(breakdown)
54
+
55
+ skill_score = breakdown.get("skill_score", 0)
56
+ semantic_score = breakdown.get("semantic_score", 0)
57
+ final_score = candidate_detail.get("final_score", 0)
58
+ tier = candidate_detail.get("tier", "")
59
+
60
+ matched = breakdown.get("matched_skills", [])
61
+ if isinstance(matched, str):
62
+ matched = json.loads(matched)
63
+ matched_names = [s.get("skill", s) if isinstance(s, dict) else s for s in matched[:6]]
64
+
65
+ below = breakdown.get("below_level_skills", [])
66
+ if isinstance(below, str):
67
+ below = json.loads(below)
68
+ below_details = []
69
+ for s in below[:4]:
70
+ if isinstance(s, dict):
71
+ below_details.append(
72
+ f"{s.get('skill','?')}: has level {s.get('candidate_level','?')}, "
73
+ f"needs level {s.get('required_level','?')}"
74
+ )
75
+
76
+ missing = breakdown.get("missing_skills", [])
77
+ if isinstance(missing, str):
78
+ missing = json.loads(missing)
79
+
80
+ flags = candidate_detail.get("flags", [])
81
+ if isinstance(flags, str):
82
+ flags = json.loads(flags)
83
+
84
+ return f"""You are an expert technical recruiter. Analyze this candidate match and write a concise, specific assessment.
85
+
86
+ JOB REQUIREMENTS:
87
+ - Role: {jd_title} ({jd_type})
88
+ - Minimum Experience: {min_years} years
89
+ - Key Required Skills: {', '.join(req_skill_names)}
90
+ - Responsibilities (sample): {'; '.join(responsibilities[:3])}
91
+
92
+ CANDIDATE PROFILE:
93
+ - Name: {name}
94
+ - Current Role: {position}
95
+ - Years of Experience: {years}
96
+ - Skills: {skills_text}
97
+ - Summary: {summary_text}
98
+ - Work History: {work_text}
99
+
100
+ MATCH ANALYSIS:
101
+ - Overall Score: {final_score:.0%} β€” {tier}
102
+ - Skills Match Score: {skill_score:.0%}
103
+ - Experience Relevance: {semantic_score:.0%}
104
+ - Matched Skills: {', '.join(matched_names) or 'None identified'}
105
+ - Below-Level Skills: {'; '.join(below_details) or 'None'}
106
+ - Missing Skills: {', '.join(missing[:5]) or 'None'}
107
+ - Flags: {', '.join(flags) or 'None'}
108
+
109
+ Write a structured assessment with EXACTLY these sections:
110
+
111
+ **Overall Fit** (2 sentences): Reference specific companies and technologies from work history. Be direct and factual.
112
+
113
+ **Strengths** (2–3 bullets): Each bullet must cite actual data β€” company names, specific technologies, durations, measurable impacts if available.
114
+
115
+ **Gaps** (1–3 bullets, skip if none): Use format: "Has [qualifier] [skill]; role requires [qualifier] [skill]". Be precise.
116
+
117
+ **Flags** (only if flags list is non-empty): Briefly explain recruiter concern for each flag.
118
+
119
+ **Recommendation**: {tier}
120
+
121
+ Be specific. No generic statements. Reference the actual data provided."""
122
+
123
+
124
+ def _fallback_explanation(candidate_detail: dict, tier: str) -> str:
125
+ name = candidate_detail.get("candidate_name", "Candidate")
126
+ position = candidate_detail.get("current_position", "")
127
+ years = candidate_detail.get("years_of_experience", 0)
128
+ score = candidate_detail.get("final_score", 0)
129
+
130
+ breakdown = candidate_detail.get("score_breakdown", {})
131
+ if isinstance(breakdown, str):
132
+ try:
133
+ breakdown = json.loads(breakdown)
134
+ except Exception:
135
+ breakdown = {}
136
+
137
+ matched = breakdown.get("matched_skills", [])
138
+ if isinstance(matched, str):
139
+ try:
140
+ matched = json.loads(matched)
141
+ except Exception:
142
+ matched = []
143
+ matched_names = [s.get("skill", s) if isinstance(s, dict) else s for s in matched[:5]]
144
+
145
+ missing = breakdown.get("missing_skills", [])
146
+ if isinstance(missing, str):
147
+ try:
148
+ missing = json.loads(missing)
149
+ except Exception:
150
+ missing = []
151
+
152
+ strengths = ", ".join(matched_names[:3]) if matched_names else "relevant technical skills"
153
+ gaps_text = f"- Missing or below-level: {', '.join(missing[:3])}" if missing else "- No critical gaps identified"
154
+
155
+ return f"""**Overall Fit**
156
+ {name} is a {tier.lower()} for this role with a {score:.0%} match score. Currently working as {position} with {years if years != -1 else 'unknown'} years of experience.
157
+
158
+ **Strengths**
159
+ - Strong in: {strengths}
160
+ - {years if years != -1 else 'Experience'} years of relevant background
161
+ - Current role aligns with core requirements
162
+
163
+ **Gaps**
164
+ {gaps_text}
165
+
166
+ **Recommendation**: {tier}
167
+
168
+ *Note: Detailed AI explanation unavailable β€” Ollama not running. Start Ollama and pull the model to enable.*"""
169
+
170
+
171
+ def generate_explanation(jd_data: dict, candidate_detail: dict) -> str:
172
+ """
173
+ Generate a human-readable explanation via Ollama cloud.
174
+ Falls back to template if Ollama is unavailable.
175
+ """
176
+ from app.config import settings
177
+
178
+ tier = candidate_detail.get("tier", "Partial Match")
179
+
180
+ try:
181
+ prompt = _build_prompt(jd_data, candidate_detail)
182
+ client = _get_client()
183
+
184
+ response = client.chat(
185
+ model=settings.ollama_model,
186
+ messages=[{"role": "user", "content": prompt}],
187
+ options={"temperature": 0.2, "num_predict": 1500},
188
+ )
189
+ text = response.message.content
190
+ return text.strip()
191
+
192
+ except Exception as e:
193
+ logger.warning(f"Ollama call failed: {e}, using fallback")
194
+ return _fallback_explanation(candidate_detail, tier)
app/services/file_parser.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Parse uploaded files:
3
+ - XLSX β†’ list of CandidateData (same column schema as Sample_Candidate_Data.xlsx)
4
+ - PDF β†’ raw text for JD parsing
5
+ """
6
+ import ast
7
+ import re
8
+ import uuid
9
+ import logging
10
+ from io import BytesIO
11
+ from typing import Optional
12
+
13
+ import pandas as pd
14
+ import fitz # PyMuPDF
15
+
16
+ from app.models.domain import CandidateData, WorkExperience
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+ # ─── Constants ────────────────────────────────────────────────────────────────
21
+
22
+ REQUIRED_COLUMNS = {
23
+ "name", "parsed_skills", "parsed_summary",
24
+ "parsed_work_experience", "parsed_metadata_calculated_years_experience",
25
+ "engineer_type", "current_position",
26
+ }
27
+
28
+ ENGINEER_TYPE_MAP = {
29
+ "ai engineer": "AI Engineer",
30
+ "ai": "AI Engineer",
31
+ "ml engineer": "AI Engineer",
32
+ "machine learning engineer": "AI Engineer",
33
+ "backend engineer": "Backend Engineer",
34
+ "backend": "Backend Engineer",
35
+ "be": "Backend Engineer",
36
+ "frontend engineer": "Frontend Engineer",
37
+ "frontend": "Frontend Engineer",
38
+ "fe": "Frontend Engineer",
39
+ "full stack engineer": "Full Stack Engineer",
40
+ "fullstack": "Full Stack Engineer",
41
+ "full stack": "Full Stack Engineer",
42
+ "software engineer": "Software Engineer",
43
+ "sde": "Software Engineer",
44
+ "data engineer": "Data Engineer",
45
+ "devops engineer": "DevOps Engineer",
46
+ "devops": "DevOps Engineer",
47
+ }
48
+
49
+ RECENT_EXP_MAP = {
50
+ "companytype.product": "Product Company",
51
+ "companytype.service": "Service Company",
52
+ "a company that build its own technology products": "Product Company",
53
+ "product": "Product Company",
54
+ "service": "Service Company",
55
+ "both": "Both",
56
+ }
57
+
58
+ GEN_AI_POSITIVE = {
59
+ "6 months - 1 year", "1 to 1.5 years", "1.5 years - 2 years",
60
+ "2+ years", "only internship/project experience",
61
+ }
62
+
63
+ GEN_AI_NEGATIVE = {"i do not have gen ai experience"}
64
+
65
+
66
+ # ─── XLSX Parsing ─────────────────────────────────────────────────────────────
67
+
68
+ def parse_xlsx(file_bytes: bytes) -> tuple[list[CandidateData], dict[str, int]]:
69
+ """Return (candidates, engineer_type_distribution)."""
70
+ df = pd.read_excel(BytesIO(file_bytes))
71
+ df.columns = [c.strip().lower().replace(" ", "_") for c in df.columns]
72
+
73
+ missing = REQUIRED_COLUMNS - set(df.columns)
74
+ if missing:
75
+ raise ValueError(f"XLSX missing required columns: {missing}")
76
+
77
+ candidates: list[CandidateData] = []
78
+ for idx, row in df.iterrows():
79
+ try:
80
+ candidate = _parse_row(row, idx)
81
+ if candidate:
82
+ candidates.append(candidate)
83
+ except Exception as e:
84
+ logger.warning(f"Skipping row {idx}: {e}")
85
+
86
+ dist: dict[str, int] = {}
87
+ for c in candidates:
88
+ dist[c.engineer_type] = dist.get(c.engineer_type, 0) + 1
89
+
90
+ return candidates, dist
91
+
92
+
93
+ def _parse_row(row: pd.Series, idx: int) -> Optional[CandidateData]:
94
+ def get(col: str, default=None):
95
+ v = row.get(col, default)
96
+ if v is None or (isinstance(v, float) and pd.isna(v)):
97
+ return default
98
+ return v
99
+
100
+ # ID
101
+ candidate_id = str(get("id") or get("submission_id") or f"cand_{idx}")
102
+
103
+ # Years
104
+ yoe = _parse_float(get("parsed_metadata_calculated_years_experience"))
105
+ if yoe is None:
106
+ yoe = _parse_float(get("years_of_experience"))
107
+ if yoe is None:
108
+ yoe = -1.0
109
+
110
+ # Engineer type
111
+ raw_type = str(get("engineer_type", "")).strip()
112
+ engineer_type = ENGINEER_TYPE_MAP.get(raw_type.lower(), raw_type or "Unknown")
113
+
114
+ # Work experience
115
+ raw_we = str(get("parsed_work_experience", "") or "")
116
+ work_experiences, we_text = _parse_work_experience(raw_we, candidate_id)
117
+
118
+ # Gen AI
119
+ gen_ai_raw = get("gen_ai_experience")
120
+ gen_ai = str(gen_ai_raw).strip() if gen_ai_raw else None
121
+
122
+ # Active looking
123
+ active_raw = get("is_actively_or_passively_looking")
124
+ if active_raw is not None:
125
+ active = str(active_raw).strip().lower() not in ("false", "no", "0", "nan", "none")
126
+ else:
127
+ active = None
128
+
129
+ flags: list[str] = []
130
+ if yoe == -1.0:
131
+ flags.append("years_unknown")
132
+ if not raw_we or raw_we == "nan":
133
+ flags.append("work_exp_unavailable")
134
+ if engineer_type == "Unknown":
135
+ flags.append("engineer_type_unknown")
136
+
137
+ return CandidateData(
138
+ id=candidate_id,
139
+ name=str(get("name", f"Candidate_{idx}")),
140
+ current_position=str(get("current_position", "")),
141
+ current_title=get("current_title"),
142
+ years_of_experience=yoe,
143
+ engineer_type=engineer_type,
144
+ gen_ai_experience=gen_ai,
145
+ parsed_skills=str(get("parsed_skills", "") or ""),
146
+ parsed_summary=str(get("parsed_summary", "") or ""),
147
+ parsed_work_experience_raw=raw_we,
148
+ parsed_work_experience_text=we_text,
149
+ work_experiences=work_experiences,
150
+ extracted_skills=[], # filled by skill_parser
151
+ most_recent_company=get("parsed_metadata_most_recent_company"),
152
+ notice_period=get("notice_period"),
153
+ is_actively_looking=active,
154
+ flags=flags,
155
+ penalties=[],
156
+ )
157
+
158
+
159
+ def _parse_work_experience(raw: str, cand_id: str) -> tuple[list[WorkExperience], str]:
160
+ if not raw or raw.strip() in ("", "nan", "None"):
161
+ return [], ""
162
+
163
+ entries: list[WorkExperience] = []
164
+ try:
165
+ parsed = ast.literal_eval("[" + raw + "]")
166
+ for item in parsed:
167
+ if isinstance(item, dict):
168
+ entries.append(WorkExperience(
169
+ company=str(item.get("company", "")),
170
+ role=str(item.get("role", "")),
171
+ duration=str(item.get("duration", "")),
172
+ description=str(item.get("description", "")),
173
+ ))
174
+ except Exception:
175
+ logger.debug(f"ast.literal_eval failed for {cand_id}, using raw text")
176
+ entries.append(WorkExperience(
177
+ company="", role="", duration="",
178
+ description=raw[:1000],
179
+ ))
180
+
181
+ text_parts = []
182
+ for e in entries:
183
+ text_parts.append(
184
+ f"{e.role} at {e.company} ({e.duration}): {e.description}"
185
+ )
186
+ return entries, " | ".join(text_parts)
187
+
188
+
189
+ def _parse_float(val) -> Optional[float]:
190
+ if val is None:
191
+ return None
192
+ try:
193
+ f = float(str(val).strip())
194
+ return f if f >= 0 else None
195
+ except (ValueError, TypeError):
196
+ return None
197
+
198
+
199
+ # ─── PDF Parsing ──────────────────────────────────────────────────────────────
200
+
201
+ def parse_pdf(file_bytes: bytes) -> str:
202
+ """Extract full text from PDF, preserving newlines."""
203
+ text_parts: list[str] = []
204
+ try:
205
+ doc = fitz.open(stream=file_bytes, filetype="pdf")
206
+ for page in doc:
207
+ text_parts.append(page.get_text("text"))
208
+ doc.close()
209
+ except Exception as e:
210
+ raise ValueError(f"Failed to read PDF: {e}")
211
+
212
+ text = "\n".join(text_parts)
213
+ # Collapse 3+ blank lines to 2
214
+ text = re.sub(r"\n{3,}", "\n\n", text)
215
+ return text.strip()
app/services/storage.py ADDED
@@ -0,0 +1,367 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Hybrid storage layer.
3
+
4
+ Candidates (100k+ rows, 50-150 MB) β†’ process memory only. Never Redis.
5
+ Reason: remote stores (Upstash free = 10 MB max request size) cannot hold this.
6
+ The raw candidate list is only needed during run_matching(); after that it can be GC'd.
7
+
8
+ Everything else (JD metadata, match status, top-N results, explanations) is tiny (<1 MB
9
+ total per session) β†’ Redis if reachable, else in-process memory fallback.
10
+
11
+ This means the system works with zero external dependencies for local dev.
12
+ """
13
+ import json
14
+ import time
15
+ import logging
16
+ from typing import Optional
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+ # TTL constants (seconds)
21
+ TTL_JD = 7 * 24 * 3600
22
+ TTL_BATCH = 7 * 24 * 3600
23
+ TTL_RESULTS = 24 * 3600
24
+ TTL_EXPLANATION = 48 * 3600
25
+
26
+
27
+ # ── In-process memory store ────────────────────────────────────────────────────
28
+
29
+ class _MemStore:
30
+ """Minimal in-process store mimicking the Redis commands we use."""
31
+
32
+ def __init__(self):
33
+ self._strings: dict[str, str] = {}
34
+ self._hashes: dict[str, dict[str, str]] = {}
35
+ self._zsets: dict[str, dict[str, float]] = {}
36
+ self._expiry: dict[str, float] = {}
37
+
38
+ def _live(self, key: str) -> bool:
39
+ exp = self._expiry.get(key)
40
+ if exp is not None and time.time() > exp:
41
+ for store in (self._strings, self._hashes, self._zsets):
42
+ store.pop(key, None)
43
+ self._expiry.pop(key, None)
44
+ return False
45
+ return True
46
+
47
+ def ping(self) -> bool:
48
+ return True
49
+
50
+ def set(self, key: str, value: str, ex: Optional[int] = None) -> None:
51
+ self._strings[key] = value
52
+ if ex:
53
+ self._expiry[key] = time.time() + ex
54
+
55
+ def get(self, key: str) -> Optional[str]:
56
+ return self._strings.get(key) if self._live(key) else None
57
+
58
+ def delete(self, key: str) -> None:
59
+ for store in (self._strings, self._hashes, self._zsets):
60
+ store.pop(key, None)
61
+ self._expiry.pop(key, None)
62
+
63
+ def expire(self, key: str, seconds: int) -> None:
64
+ self._expiry[key] = time.time() + seconds
65
+
66
+ def hset(self, key: str, mapping: dict) -> None:
67
+ if key not in self._hashes:
68
+ self._hashes[key] = {}
69
+ self._hashes[key].update(mapping)
70
+
71
+ def hgetall(self, key: str) -> dict:
72
+ return dict(self._hashes.get(key, {})) if self._live(key) else {}
73
+
74
+ def keys(self, pattern: str) -> list[str]:
75
+ prefix = pattern.replace("*", "")
76
+ result = set()
77
+ for store in (self._strings, self._hashes, self._zsets):
78
+ for k in list(store.keys()):
79
+ if k.startswith(prefix) and self._live(k):
80
+ result.add(k)
81
+ return list(result)
82
+
83
+ def zadd(self, key: str, mapping: dict) -> None:
84
+ if key not in self._zsets:
85
+ self._zsets[key] = {}
86
+ self._zsets[key].update(mapping)
87
+
88
+ def zrevrange(self, key: str, start: int, stop: int,
89
+ withscores: bool = False) -> list:
90
+ if not self._live(key):
91
+ return []
92
+ items = sorted(self._zsets.get(key, {}).items(),
93
+ key=lambda x: x[1], reverse=True)
94
+ sliced = items[start:] if stop == -1 else items[start: stop + 1]
95
+ return sliced if withscores else [m for m, _ in sliced]
96
+
97
+ def pipeline(self):
98
+ return _MemPipeline(self)
99
+
100
+
101
+ class _MemPipeline:
102
+ """Fake pipeline that buffers calls and executes them synchronously."""
103
+
104
+ def __init__(self, store: _MemStore):
105
+ self._store = store
106
+ self._ops: list = []
107
+
108
+ def __getattr__(self, name):
109
+ def recorder(*args, **kwargs):
110
+ self._ops.append((name, args, kwargs))
111
+ return self
112
+ return recorder
113
+
114
+ def execute(self):
115
+ results = []
116
+ for name, args, kwargs in self._ops:
117
+ fn = getattr(self._store, name)
118
+ results.append(fn(*args, **kwargs))
119
+ self._ops.clear()
120
+ return results
121
+
122
+
123
+ _mem = _MemStore()
124
+
125
+
126
+ # ── Redis client (optional) ────────────────────────────────────────────────────
127
+
128
+ _redis_client = None
129
+ _redis_ok: Optional[bool] = None # None = not yet tested
130
+
131
+
132
+ def _store():
133
+ """Return Redis client if reachable, else in-process _MemStore."""
134
+ global _redis_client, _redis_ok
135
+
136
+ if _redis_ok is False:
137
+ return _mem
138
+
139
+ try:
140
+ if _redis_client is None:
141
+ from app.config import settings
142
+ import redis as redis_lib
143
+ _redis_client = redis_lib.from_url(
144
+ settings.redis_url,
145
+ decode_responses=True,
146
+ socket_connect_timeout=3,
147
+ socket_timeout=5,
148
+ ssl_cert_reqs=None,
149
+ )
150
+ _redis_client.ping()
151
+ _redis_ok = True
152
+ return _redis_client
153
+ except Exception as e:
154
+ if _redis_ok is not False:
155
+ logger.warning(f"Redis unavailable ({e}), using in-process memory store")
156
+ _redis_ok = False
157
+ return _mem
158
+
159
+
160
+ def ping() -> bool:
161
+ try:
162
+ return _store().ping()
163
+ except Exception:
164
+ return False
165
+
166
+
167
+ def _now_iso() -> str:
168
+ from datetime import datetime, timezone
169
+ return datetime.now(timezone.utc).isoformat()
170
+
171
+
172
+ # ── Candidates: process memory ONLY ───────────────────────────────────────────
173
+ # 100k candidates β‰ˆ 50-150 MB JSON. Remote stores cannot handle this cheaply.
174
+ # After matching, only the top-N detail hashes are persisted; this list is freed.
175
+
176
+ _candidate_mem: dict[str, str] = {} # batch_id -> candidates_json
177
+ _candidate_exp: dict[str, float] = {} # batch_id -> expiry unix timestamp
178
+
179
+
180
+ def _cand_live(batch_id: str) -> bool:
181
+ exp = _candidate_exp.get(batch_id)
182
+ if exp is not None and time.time() > exp:
183
+ _candidate_mem.pop(batch_id, None)
184
+ _candidate_exp.pop(batch_id, None)
185
+ return False
186
+ return batch_id in _candidate_mem
187
+
188
+
189
+ # ── JD ────────────────────────────────────────────────────────────────────────
190
+
191
+ def save_jd(jd_id: str, data: dict) -> None:
192
+ r = _store()
193
+ key = f"jd:{jd_id}"
194
+ r.hset(key, mapping={
195
+ k: json.dumps(v) if not isinstance(v, str) else v
196
+ for k, v in data.items()
197
+ })
198
+ r.expire(key, TTL_JD)
199
+
200
+
201
+ def get_jd(jd_id: str) -> Optional[dict]:
202
+ raw = _store().hgetall(f"jd:{jd_id}")
203
+ if not raw:
204
+ return None
205
+ result = {}
206
+ for k, v in raw.items():
207
+ try:
208
+ result[k] = json.loads(v)
209
+ except (json.JSONDecodeError, TypeError):
210
+ result[k] = v
211
+ return result
212
+
213
+
214
+ def list_jds() -> list[dict]:
215
+ r = _store()
216
+ jds = []
217
+ for key in sorted(r.keys("jd:*"), reverse=True):
218
+ jd_id = key.split(":", 1)[1]
219
+ data = get_jd(jd_id)
220
+ if data:
221
+ jds.append(data)
222
+ return jds
223
+
224
+
225
+ # ── Candidate batch ───────────────────────────────────────────────────────────
226
+
227
+ def save_batch(batch_id: str, metadata: dict, candidates_json: str) -> None:
228
+ # Metadata (tiny) β†’ Redis / mem store
229
+ r = _store()
230
+ meta_key = f"batch:{batch_id}"
231
+ r.hset(meta_key, mapping={
232
+ k: json.dumps(v) if not isinstance(v, str) else v
233
+ for k, v in metadata.items()
234
+ })
235
+ r.expire(meta_key, TTL_BATCH)
236
+
237
+ # Candidates (large) β†’ process memory only, never remote
238
+ _candidate_mem[batch_id] = candidates_json
239
+ _candidate_exp[batch_id] = time.time() + TTL_BATCH
240
+ logger.info(f"Stored {len(candidates_json) // 1024} KB candidates for batch {batch_id} in process memory")
241
+
242
+
243
+ def get_batch_metadata(batch_id: str) -> Optional[dict]:
244
+ raw = _store().hgetall(f"batch:{batch_id}")
245
+ if not raw:
246
+ return None
247
+ result = {}
248
+ for k, v in raw.items():
249
+ try:
250
+ result[k] = json.loads(v)
251
+ except (json.JSONDecodeError, TypeError):
252
+ result[k] = v
253
+ return result
254
+
255
+
256
+ def get_batch_candidates_json(batch_id: str) -> Optional[str]:
257
+ if not _cand_live(batch_id):
258
+ return None
259
+ return _candidate_mem.get(batch_id)
260
+
261
+
262
+ def list_batches() -> list[dict]:
263
+ r = _store()
264
+ batches = []
265
+ for key in sorted(r.keys("batch:*"), reverse=True):
266
+ batch_id = key.split(":", 1)[1]
267
+ data = get_batch_metadata(batch_id)
268
+ if data:
269
+ batches.append(data)
270
+ return batches
271
+
272
+
273
+ # ── Match status ──────────────────────────────────────────────────────────────
274
+
275
+ def set_match_status(
276
+ match_id: str,
277
+ status: str,
278
+ progress: Optional[float] = None,
279
+ message: Optional[str] = None,
280
+ ) -> None:
281
+ r = _store()
282
+ key = f"match_status:{match_id}"
283
+ data: dict[str, str] = {"status": status, "match_id": match_id}
284
+ if progress is not None:
285
+ data["progress"] = str(progress)
286
+ if message:
287
+ data["message"] = message
288
+ if status == "completed":
289
+ data["completed_at"] = _now_iso()
290
+ r.hset(key, mapping=data)
291
+ r.expire(key, TTL_RESULTS)
292
+
293
+
294
+ def get_match_status(match_id: str) -> Optional[dict]:
295
+ raw = _store().hgetall(f"match_status:{match_id}")
296
+ return raw if raw else None
297
+
298
+
299
+ # ── Match results ─────────────────────────────────────────────────────────────
300
+
301
+ def save_match_results(jd_id: str, batch_id: str, results: list[dict]) -> None:
302
+ r = _store()
303
+ pipe = r.pipeline()
304
+
305
+ zset_key = f"results:{jd_id}:{batch_id}"
306
+ pipe.delete(zset_key)
307
+
308
+ for result in results:
309
+ cid = result["candidate_id"]
310
+ score = result["final_score"]
311
+ pipe.zadd(zset_key, {cid: score})
312
+
313
+ detail_key = f"detail:{jd_id}:{batch_id}:{cid}"
314
+ flat = {k: json.dumps(v) if not isinstance(v, str) else v
315
+ for k, v in result.items()}
316
+ pipe.hset(detail_key, mapping=flat)
317
+ pipe.expire(detail_key, TTL_RESULTS)
318
+
319
+ pipe.expire(zset_key, TTL_RESULTS)
320
+ pipe.execute()
321
+
322
+
323
+ def get_ranked_candidates(
324
+ jd_id: str, batch_id: str, top_n: int
325
+ ) -> list[tuple[str, float]]:
326
+ return _store().zrevrange(
327
+ f"results:{jd_id}:{batch_id}", 0, top_n - 1, withscores=True
328
+ )
329
+
330
+
331
+ def get_candidate_detail(
332
+ jd_id: str, batch_id: str, candidate_id: str
333
+ ) -> Optional[dict]:
334
+ raw = _store().hgetall(f"detail:{jd_id}:{batch_id}:{candidate_id}")
335
+ if not raw:
336
+ return None
337
+ result = {}
338
+ for k, v in raw.items():
339
+ try:
340
+ result[k] = json.loads(v)
341
+ except (json.JSONDecodeError, TypeError):
342
+ result[k] = v
343
+ return result
344
+
345
+
346
+ # ── Explanation cache ─────────────────────────────────────────────────────────
347
+
348
+ def get_explanation(jd_id: str, batch_id: str, candidate_id: str) -> Optional[str]:
349
+ return _store().get(f"explanation:{jd_id}:{batch_id}:{candidate_id}")
350
+
351
+
352
+ def save_explanation(jd_id: str, batch_id: str, candidate_id: str, text: str) -> None:
353
+ _store().set(
354
+ f"explanation:{jd_id}:{batch_id}:{candidate_id}",
355
+ text,
356
+ ex=TTL_EXPLANATION,
357
+ )
358
+
359
+
360
+ # ── Active batch pointer ──────────────────────────────────────────────────────
361
+
362
+ def set_active_batch(jd_id: str, batch_id: str) -> None:
363
+ _store().set(f"active_batch:{jd_id}", batch_id, ex=TTL_RESULTS)
364
+
365
+
366
+ def get_active_batch(jd_id: str) -> Optional[str]:
367
+ return _store().get(f"active_batch:{jd_id}")
fly.toml ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # fly.toml app configuration file generated for coderound-matching-api
2
+ #
3
+ # See https://fly.io/docs/reference/configuration/ for information about how to use this file.
4
+ #
5
+
6
+ app = "coderound-matching-api"
7
+ primary_region = "iad"
8
+
9
+ [build]
10
+
11
+ # Environment variables are set as secrets in Fly.io
12
+ # Use `fly secrets set KEY=VALUE` to set them.
13
+ # Example:
14
+ # fly secrets set REDIS_URL="redis://user:pass@host:port"
15
+ # fly secrets set CORS_ORIGINS="http://localhost:3000,https://your-frontend.com"
16
+ # fly secrets set OLLAMA_HOST="https://your-ollama-instance.com" OLLAMA_API_KEY="your-key"
17
+
18
+ [env]
19
+ PORT = "8000"
20
+ # Default log level, can be overridden by secrets
21
+ LOG_LEVEL = "INFO"
22
+ # Default embedding model, can be overridden by secrets
23
+ EMBEDDING_MODEL = "all-MiniLM-L6-v2"
24
+
25
+ [http_service]
26
+ internal_port = 8000
27
+ force_https = true
28
+ auto_stop_machines = "off"
29
+ auto_start_machines = true
30
+ min_machines_running = 1
31
+ processes = ["app"]
32
+
33
+ [[vm]]
34
+ memory = "2gb"
35
+ cpu_kind = "shared"
36
+ cpus = 2
37
+
38
+ # No persistent volume is needed as state is managed by Redis.
39
+ # [mounts]
40
+
41
+ [[services]]
42
+ protocol = "tcp"
43
+ internal_port = 8000
44
+
45
+ [[services.ports]]
46
+ port = 80
47
+ handlers = ["http"]
48
+ force_https = true
49
+
50
+ [[services.ports]]
51
+ port = 443
52
+ handlers = ["tls", "http"]
53
+
54
+ [services.concurrency]
55
+ type = "connections"
56
+ hard_limit = 25
57
+ soft_limit = 20
58
+
59
+ # Health check to ensure the service is responsive
60
+ [[services.http_checks]]
61
+ interval = "30s"
62
+ timeout = "5s"
63
+ grace_period = "60s"
64
+ method = "get"
65
+ path = "/health"
66
+ protocol = "http"
requirements.txt ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fastapi==0.115.5
2
+ uvicorn[standard]==0.32.1
3
+ python-multipart==0.0.12
4
+ pydantic
5
+ pydantic-settings
6
+ redis==5.2.1
7
+ pandas==2.2.3
8
+ openpyxl==3.1.5
9
+ pymupdf==1.25.1
10
+ sentence-transformers==3.3.1
11
+ rank-bm25==0.2.2
12
+ numpy==1.26.4
13
+ python-dotenv==1.0.1
14
+ httpx==0.28.1
15
+ ollama==0.6.2