Deploy Redrob Streamlit sandbox

README.md

@@ -1,20 +1,43 @@
 ---
-title: RedrobJobRanker
-emoji: 🚀
-colorFrom: red
-colorTo: red
-sdk: docker
-app_port: 8501
-tags:
-- streamlit
+title: Redrob Candidate Intelligence Engine
+emoji: "🚀"
+colorFrom: green
+colorTo: blue
+sdk: streamlit
+sdk_version: 1.30.0
+app_file: app.py
 pinned: false
-short_description: Streamlit template space
 license: mit
 ---
 
-# Welcome to Streamlit!
+# Redrob Candidate Intelligence — Stage 1 Sandbox
 
-Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
+This Space hosts the **Stage 1 sandbox** for the Redrob hackathon
+submission (Stage-1 requirement per submission spec Section 10.5).
 
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
+The Streamlit app accepts a candidate sample (≤5,000 records) uploaded
+by the user, runs the **complete pipeline end-to-end** (BM25 + 5-axis
+PageRank + 74 features + deterministic composite + reasoning), and
+produces a downloadable ranked CSV.
+
+For the **full 100k run**, the official reproduce command is:
+```bash
+python scripts/run_ranking.py --no_dense --no_lgbm
+```
+
+## What's in this Space
+
+- `app.py` — Streamlit sandbox UI
+- `redrob/` — full pipeline package (config, blueprint, retrieval,
+  graph, features, rank, reasoning, submit)
+- `scripts/run_ranking.py` — the official pipeline (re-use from app.py)
+- `requirements.txt` — Python dependencies
+
+## Source repository
+
+The complete submission (top-100 ranked CSV, full source code, all
+experiments) is at:
+**https://github.com/AdityaProCoder/JobRanker**
+
+`submission.csv` (top-100 ranked) and `submission_metadata.yaml` are
+in the GitHub repo.

redrob/__init__.py

@@ -0,0 +1,2 @@
+"""Redrob Candidate Intelligence Engine — top-level package."""
+__version__ = "1.0.0"

redrob/blueprint.py

@@ -0,0 +1,56 @@
+"""Role Blueprint extraction.
+
+The blueprint is largely static (already curated in `config.py`) but the
+public entry point also writes `artifacts/blueprint.json` for the sandbox
+to display.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List
+
+from . import config
+
+
+def build_blueprint() -> Dict[str, Any]:
+    """Return the role blueprint as a plain dict (also JSON-serializable)."""
+    bp: Dict[str, Any] = {
+        "role": "Senior AI Engineer — Redrob AI (Series A)",
+        "company": "Redrob AI",
+        "location_preference": "Pune / Noida (Hybrid)",
+        "country": "India",
+        "experience_band": {
+            "min": config.YOE_MIN,
+            "ideal_low": config.YOE_IDEAL_LOW,
+            "ideal_high": config.YOE_IDEAL_HIGH,
+            "max_useful": config.YOE_MAX_USEFUL,
+        },
+        "core_competencies": sorted(config.CORE_COMPETENCIES),
+        "adjacent_competencies": sorted(config.ADJACENT_COMPETENCIES),
+        "negative_competencies": sorted(config.NEGATIVE_COMPETENCIES),
+        "target_titles": sorted(config.TARGET_TITLES),
+        "title_role_groups": config.TITLE_ROLE_GROUPS,
+        "title_group_weight": config.TITLE_GROUP_WEIGHT,
+        "notice_period_ok_days": config.NOTICE_OK_DAYS,
+        "notice_period_hard_days": config.NOTICE_HARD_DAYS,
+        "preferred_locations": sorted(config.PREFERRED_LOCATIONS),
+        "country_ok": sorted(config.COUNTRY_OK),
+        "it_services_pure_play": sorted(config.IT_SERVICES_PURE_PLAY),
+        "product_company_hints": sorted(config.PRODUCT_COMPANY_HINTS),
+        "query_terms": list(config.JD_QUERY_TERMS),
+        "structured_gate_terms": sorted(config.STRUCTURED_GATE_TERMS),
+    }
+    return bp
+
+
+def save_blueprint(path=None) -> Dict[str, Any]:
+    bp = build_blueprint()
+    out = config.BLUEPRINT_JSON if path is None else path
+    with open(out, "w", encoding="utf-8") as f:
+        json.dump(bp, f, indent=2, ensure_ascii=False)
+    return bp
+
+
+if __name__ == "__main__":
+    save_blueprint()
+    print(f"blueprint -> {config.BLUEPRINT_JSON}")

redrob/config.py

@@ -0,0 +1,413 @@
+"""Project-wide configuration.
+
+Paths, JD blueprint, and tuning knobs. Editable from a single place.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Dict, List, Set
+
+
+# ---------------------------------------------------------------------------
+# Paths
+# ---------------------------------------------------------------------------
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+ARTIFACTS_DIR = PROJECT_ROOT / "artifacts"
+MODELS_DIR = ARTIFACTS_DIR / "models"
+ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True)
+MODELS_DIR.mkdir(parents=True, exist_ok=True)
+
+DEFAULT_CANDIDATES_JSONL = (
+    PROJECT_ROOT
+    / "[PUB] India_runs_data_and_ai_challenge"
+    / "[PUB] India_runs_data_and_ai_challenge"
+    / "India_runs_data_and_ai_challenge"
+    / "candidates.jsonl"
+)
+DEFAULT_JOB_DESCRIPTION = (
+    PROJECT_ROOT
+    / "[PUB] India_runs_data_and_ai_challenge"
+    / "[PUB] India_runs_data_and_ai_challenge"
+    / "India_runs_data_and_ai_challenge"
+    / "job_description.docx"
+)
+DEFAULT_OUTPUT_CSV = PROJECT_ROOT / "submission.csv"
+DEFAULT_VALIDATOR = (
+    PROJECT_ROOT
+    / "[PUB] India_runs_data_and_ai_challenge"
+    / "[PUB] India_runs_data_and_ai_challenge"
+    / "India_runs_data_and_ai_challenge"
+    / "validate_submission.py"
+)
+
+# Artifact filenames
+CANDIDATES_PARQUET = ARTIFACTS_DIR / "candidates.parquet"
+GRAPH_PICKLE = ARTIFACTS_DIR / "graph.pkl"
+TITLE_EMB_NPY = ARTIFACTS_DIR / "title_embeddings.npy"
+BM25_PICKLE = ARTIFACTS_DIR / "bm25_index.pkl"
+BM25_TOKENS_PICKLE = ARTIFACTS_DIR / "bm25_tokens.pkl"
+DENSE_NPY = ARTIFACTS_DIR / "dense_index.npy"
+DENSE_IDS_JSON = ARTIFACTS_DIR / "dense_ids.json"
+RANKER_TXT = ARTIFACTS_DIR / "ranker.txt"
+BLUEPRINT_JSON = ARTIFACTS_DIR / "blueprint.json"
+
+# ---------------------------------------------------------------------------
+# Role Blueprint — Senior AI Engineer (Redrob founding team)
+# Extracted from job_description.docx. Kept here in code so the sandbox
+# can run without parsing the .docx and so the blueprint is auditable.
+# ---------------------------------------------------------------------------
+
+CORE_COMPETENCIES: Set[str] = {
+    # ML foundations
+    "PyTorch", "TensorFlow", "Transformers", "Hugging Face", "HuggingFace",
+    "LLM", "LLMs", "Large Language Models", "Fine-tuning", "Fine-Tuning",
+    "LoRA", "QLoRA", "PEFT", "Prompt Engineering",
+    # Retrieval / ranking infra
+    "RAG", "Retrieval Augmented Generation", "Vector Database", "Vector Databases",
+    "Pinecone", "Weaviate", "Qdrant", "Milvus", "FAISS", "OpenSearch",
+    "Elasticsearch", "BM25", "Hybrid Search", "Dense Retrieval",
+    "Sentence Transformers", "sentence-transformers", "BGE", "E5",
+    "Embeddings", "Semantic Search",
+    # IR / ranking theory
+    "Learning to Rank", "Learning-to-Rank", "Learning2Rank", "LTR",
+    "XGBoost", "LightGBM", "CatBoost", "NDCG", "MRR", "MAP",
+    "LambdaRank", "LambdaMART",
+    # Engineering
+    "Python", "AWS", "GCP", "Azure", "Docker", "Kubernetes",
+    "Distributed Systems", "MLOps", "ML Ops", "Inference",
+    # Eval / experimentation
+    "A/B Testing", "A/B Test", "Experimentation", "Evaluation",
+    "Offline Evaluation", "Online Evaluation", "Eval Framework", "Evaluation Framework",
+}
+
+ADJACENT_COMPETENCIES: Set[str] = {
+    "Spark", "PySpark", "Airflow", "Kafka", "dbt", "Snowflake",
+    "Databricks", "BigQuery", "Redshift", "Beam", "Flink",
+    "Feature Store", "Feature Engineering", "MLflow", "Kubeflow",
+    "Triton", "TensorRT", "ONNX", "RecSys", "Recommendation",
+    "Recommendation Systems", "Search", "Ranking", "Personalization",
+    "Open Source", "Open-Source", "GitHub", "NLP", "Computer Vision",
+    "LangChain", "LlamaIndex", "LangGraph",
+}
+
+NEGATIVE_COMPETENCIES: Set[str] = {
+    # these are not disqualifiers by themselves — used to detect
+    # "Marketing Manager + AI keywords" profiles during honeypot scoring
+    "SEO", "Photoshop", "Illustrator", "Content Writing", "Sales",
+    "Marketing", "Accounting", "Mechanical Design", "Civil Engineering",
+    "Customer Support", "Operations Management", "HR Management",
+    "Business Analysis", "Six Sigma", "SAP", "Recruiting",
+    "Brand Design", "Talent Acquisition", "CAD", "SolidWorks",
+    "ANSYS", "FEA", "DFM", "DFMA",
+}
+
+# Phrases that the structured high-recall gate uses to keep adjacent roles
+# like Search/Relevance Engineers in the pool.
+STRUCTURED_GATE_TERMS: Set[str] = {
+    "pytorch", "tensorflow", "hugging", "transformers", "llm", "rag",
+    "retrieval", "recommendation", "ranking", "ranker", "search", "embedding",
+    "vector", "faiss", "pinecone", "weaviate", "qdrant", "milvus",
+    "opensearch", "elasticsearch", "bm25", "xgboost", "lightgbm", "catboost",
+    "learning to rank", "learning-to-rank", "learning2rank", "lambdarank",
+    "lora", "qlora", "peft", "fine-tuning", "fine tuning", "fine-tune",
+    "sentence-transformer", "sentence transformer", "bge", "e5",
+    "ml engineer", "ai engineer", "applied scientist", "applied ml",
+    "machine learning", "data scientist", "nlp engineer", "nlp",
+    "search engineer", "search relevance", "relevance engineer",
+    "recommender", "recommendation systems", "ranker", "rank model",
+    "feature store", "feature engineering", "mlops", "kubeflow", "mlflow",
+    "triton", "tensorrt", "onnx", "distributed training", "inference",
+    "hybrid search", "dense retrieval", "sparse retrieval",
+    "promotion velocity", "reranking", "re-ranking", "rerank", "re-rank",
+    "vector database", "vector search",
+}
+
+TARGET_TITLES: Set[str] = {
+    "ML Engineer", "Senior ML Engineer", "Staff ML Engineer", "Principal ML Engineer",
+    "Machine Learning Engineer", "Senior Machine Learning Engineer",
+    "AI Engineer", "Senior AI Engineer", "Staff AI Engineer",
+    "Applied Scientist", "Senior Applied Scientist", "Staff Applied Scientist",
+    "Applied ML Engineer", "Senior Applied ML Engineer",
+    "Research Engineer", "Senior Research Engineer",
+    "Recommendation Systems Engineer", "Search Engineer", "Search Relevance Engineer",
+    "Relevance Engineer", "Ranking Engineer", "Senior Ranking Engineer",
+    "NLP Engineer", "Senior NLP Engineer",
+    "Data Scientist", "Senior Data Scientist", "Staff Data Scientist",
+    "Machine Learning Scientist", "Senior Machine Learning Scientist",
+    "AI Specialist", "Senior AI Specialist",
+    "Founding Engineer", "Founding ML Engineer",
+}
+
+# Title similarity groups — used for the graduated title fit feature.
+# Each group has a target role weight; matches accumulate.
+TITLE_ROLE_GROUPS: Dict[str, List[str]] = {
+    "applied_ml": [
+        "ml engineer", "machine learning engineer", "ai engineer",
+        "applied scientist", "applied ml", "research engineer",
+        "machine learning scientist", "ai specialist",
+    ],
+    "retrieval_ranking": [
+        "search engineer", "search relevance engineer", "relevance engineer",
+        "ranking engineer", "recommendation systems engineer",
+        "recommender", "recommendation systems",
+    ],
+    "nlp_llm": [
+        "nlp engineer", "nlp", "llm engineer", "generative ai engineer",
+    ],
+    "data_science": [
+        "data scientist", "senior data scientist", "applied data scientist",
+    ],
+    "data_platform": [
+        "data engineer", "senior data engineer", "analytics engineer",
+        "ml platform engineer", "ml infrastructure engineer", "mlops engineer",
+    ],
+    "generic_swe": [
+        "software engineer", "senior software engineer", "backend engineer",
+        "full stack developer", "full stack engineer", "platform engineer",
+    ],
+    "non_target": [
+        "marketing manager", "hr manager", "accountant", "sales executive",
+        "business analyst", "operations manager", "project manager",
+        "graphic designer", "content writer", "civil engineer",
+        "mechanical engineer", "customer support", "qa engineer",
+        "frontend engineer", "frontend developer", "java developer",
+        ".net developer", "mobile developer", "devops engineer",
+        "cloud engineer", "data analyst", "frontend",
+    ],
+}
+
+# Title weights used by the graduated title fit scorer (target role weight).
+# Updated based on data analysis: Senior Data Scientist with strong JD-core
+# skills (BM25, FAISS, NDCG) at Microsoft/Google is a great fit. Boosted
+# data_science weight from 0.70 to 0.85. Also added product_company tier
+# boosts to capture additional strong-elite candidates.
+TITLE_GROUP_WEIGHT: Dict[str, float] = {
+    "applied_ml": 1.00,
+    "retrieval_ranking": 0.95,
+    "nlp_llm": 0.90,
+    "data_science": 0.85,  # boosted: Sr DS w/ strong JD-core skills is a great fit
+    "data_platform": 0.60,
+    "generic_swe": 0.45,
+    "non_target": 0.05,
+}
+
+# Seniority band (years of experience).
+YOE_MIN = 4.0
+YOE_IDEAL_LOW = 5.0
+YOE_IDEAL_HIGH = 9.0
+YOE_MAX_USEFUL = 15.0
+
+# Notice period & location
+NOTICE_OK_DAYS = 30
+NOTICE_HARD_DAYS = 90
+PREFERRED_LOCATIONS = {
+    "Noida", "Pune", "Hyderabad", "Mumbai", "Bangalore", "Bengaluru",
+    "Delhi", "Gurgaon", "Gurugram", "Delhi NCR",
+}
+COUNTRY_OK = {"India"}
+
+# Companies whose culture the JD flags ("title-chasers", "framework enthusiasts",
+# pure-services). Not strict exclusions but used as a soft penalty in features.
+IT_SERVICES_PURE_PLAY = {
+    "TCS", "Infosys", "Wipro", "HCL", "Tech Mahindra", "Cognizant",
+    "Capgemini", "Accenture", "Mindtree", "L&T Infotech", "Mphasis",
+    "Persistent", "Zensar", "Hexaware", "Birlasoft", "Cyient",
+}
+PRODUCT_COMPANY_HINTS = {
+    "Razorpay", "Zerodha", "CRED", "PhonePe", "Paytm", "Swiggy",
+    "Zomato", "Ola", "Flipkart", "Meesho", "Urban Company",
+    "Dream11", "Groww", "Cars24", "MPL", "Lenskart", "Nykaa",
+    "Postman", "Freshworks", "Zoho", "Chargebee", "Razorpay",
+    "BrowserStack", "Sprinklr", "CleverTap", "MoEngage", "ShareChat",
+    "InMobi", "Oyo", "OYO", "Rivigo", "Delhivery", "Udaan",
+    "Microsoft", "Google", "Meta", "Amazon", "Apple", "Netflix",
+    "Stripe", "Airbnb", "Uber", "Linkedin", "LinkedIn",
+    "Salesforce", "Adobe", "Nvidia", "NVIDIA", "Intel", "IBM",
+    "Oracle", "SAP", "ServiceNow", "Snowflake", "Databricks",
+    "Redrob", "Redrob AI", "RedrobAI",
+    "OpenAI", "Anthropic", "Cohere", "Hugging Face", "HuggingFace",
+    "Pinecone", "Weaviate", "Qdrant",
+    # Indian AI-first labs / platforms (extended coverage)
+    "Sarvam", "Sarvam AI", "Niramai", "Rephrase", "Fractal",
+    "Yellow.ai", "Mad Street Den", "Wysa", "Haptik",
+    "Vernacular.ai", "Agnatha", "Aganitha", "Uniphore", "Observe.AI",
+    "Turing", "Turing.com", "Locobuzz", "Verloop", "Verloop.io",
+    "Bobble", "Bobble AI", "Galytix", "Locus", "Locus.sh",
+    "Conv.AI", "Yellow Messenger", "Krutrim",
+    # Ed-tech / health-tech (product companies with AI teams)
+    "BYJU", "BYJUS", "BYJU'S", "Unacademy", "Vedantu", "upGrad",
+    "upGrad", "Simplilearn", "Great Learning", "Eruditus",
+    "PolicyBazaar", "PolicyBazaar.com", "Lendingkart",
+}
+
+# ---------------------------------------------------------------------------
+# Retrieval knobs
+# ---------------------------------------------------------------------------
+
+BM25_TOP_K = 3000
+DENSE_TOP_K = 3000
+STRUCTURED_GATE_TOP_K = 8000   # recall-friendly; not a hard filter
+RRF_K = 60
+SHORTLIST_N = 4000
+
+# DENSE_MODEL can be overridden via env var REDROB_DENSE_MODEL
+# Default uses MiniLM for fast offline runs; stronger BGE models are
+# supported by setting the env var before running.
+DEFAULT_DENSE_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+# Stronger alternatives (require the model to be cached locally):
+#   "BAAI/bge-small-en-v1.5"
+#   "BAAI/bge-base-en-v1.5"
+#   "BAAI/bge-large-en-v1.5"
+DENSE_MODEL_OPTIONS = [
+    DEFAULT_DENSE_MODEL,
+    "BAAI/bge-small-en-v1.5",
+    "BAAI/bge-base-en-v1.5",
+    "BAAI/bge-large-en-v1.5",
+]
+
+# BM25 / dense query
+JD_QUERY_TERMS = [
+    "Senior AI Engineer",
+    "Applied ML Engineer",
+    "retrieval augmented generation",
+    "vector search",
+    "hybrid search",
+    "BM25 FAISS Pinecone",
+    "PyTorch transformers",
+    "fine-tuning LoRA QLoRA PEFT",
+    "learning to rank XGBoost LightGBM",
+    "NDCG MRR MAP",
+    "embeddings sentence-transformers BGE E5",
+    "vector database Milvus Qdrant Weaviate",
+    "production machine learning systems",
+    "evaluation framework A/B testing",
+    "ranker ranking recommendation",
+    "MLOps inference distributed",
+]
+
+# ---------------------------------------------------------------------------
+# Ranker knobs
+# ---------------------------------------------------------------------------
+
+RANKER_PARAMS = dict(
+    objective="lambdarank",
+    metric="ndcg",
+    eval_at=[10, 50],
+    num_leaves=63,
+    learning_rate=0.05,
+    n_estimators=600,
+    lambdarank_truncation_level=50,
+    min_data_in_leaf=20,
+    feature_fraction=0.85,
+    bagging_fraction=0.85,
+    bagging_freq=5,
+    random_state=42,
+    verbose=-1,
+)
+
+# Hard exclude honeypots at/above this penalty.
+HONEYPOT_HARD_EXCLUDE = 0.5
+
+# Reasoning template rotation
+REASONING_TEMPLATE_COUNT = 15
+
+# ---------------------------------------------------------------------------
+# JD-criticality weighting (for skill feature engineering)
+# ---------------------------------------------------------------------------
+
+JD_CRITICAL: set[str] = {
+    # The JD's "absolutely need" — weight 2.0
+    "RAG", "Retrieval Augmented Generation",
+    "FAISS", "Pinecone", "Weaviate", "Qdrant", "Milvus",
+    "OpenSearch", "Elasticsearch",
+    "BM25", "Hybrid Search", "Dense Retrieval",
+    "Sentence Transformers", "BGE", "E5", "Embeddings",
+    "Learning to Rank", "LambdaRank", "NDCG", "MRR", "MAP",
+    "XGBoost", "LightGBM", "CatBoost",
+    "PyTorch", "Transformers", "LLM", "Fine-tuning", "LoRA", "QLoRA", "PEFT",
+    "Prompt Engineering", "Hugging Face", "HuggingFace",
+    "Vector Database", "Vector Databases",
+}
+JD_NICE_TO_HAVE: set[str] = {
+    # JD "nice to have" — weight 1.0
+    "Kubernetes", "Docker", "AWS", "GCP", "Azure", "Python",
+    "MLOps", "ML Ops", "MLflow", "Kubeflow",
+    "Triton", "TensorRT", "ONNX",
+    "Distributed Systems", "Inference", "Evaluation",
+    "A/B Testing", "A/B Test", "Experimentation",
+    "NLP", "Computer Vision",
+    "Recommendation", "Recommendation Systems", "Search", "Ranking",
+    "RecSys", "LlamaIndex", "LangChain", "LangGraph",
+    "Open Source", "Open-Source",
+}
+
+# ---------------------------------------------------------------------------
+# Multi-axis PPR: 5 JD axes
+# ---------------------------------------------------------------------------
+
+JD_AXES: dict[str, list[str]] = {
+    "applied_ml": [
+        "PyTorch", "TensorFlow", "Transformers", "Hugging", "LLM", "LoRA",
+        "QLoRA", "PEFT", "Fine-tuning", "Fine-Tuning", "Prompt Engineering",
+        "NLP", "scikit-learn",
+    ],
+    "retrieval_rank": [
+        "FAISS", "Pinecone", "Weaviate", "Qdrant", "Milvus", "OpenSearch",
+        "Elasticsearch", "BM25", "Hybrid Search", "Dense Retrieval",
+        "Learning to Rank", "LambdaRank", "NDCG", "MRR", "MAP",
+        "XGBoost", "LightGBM", "CatBoost",
+        "Sentence Transformers", "BGE", "E5", "Embeddings",
+    ],
+    "nlp_llm": [
+        "LLM", "Fine-tuning", "LoRA", "QLoRA", "PEFT", "Prompt Engineering",
+        "RAG", "Retrieval Augmented Generation", "Vector Database",
+        "LangChain", "LlamaIndex", "LangGraph", "NLP", "Transformers",
+    ],
+    "production_eng": [
+        "Kubernetes", "Docker", "AWS", "GCP", "Azure", "Distributed Systems",
+        "MLOps", "MLflow", "Kubeflow", "Triton", "TensorRT", "ONNX",
+        "Inference", "Evaluation",
+    ],
+    "product_company": [
+        "Razorpay", "Zerodha", "CRED", "PhonePe", "Paytm", "Swiggy", "Zomato",
+        "Ola", "Flipkart", "Meesho", "Urban Company", "Dream11", "Groww",
+        "Cars24", "Postman", "Freshworks", "Zoho", "Chargebee", "BrowserStack",
+        "Sprinklr", "CleverTap", "MoEngage", "ShareChat", "InMobi",
+        "Microsoft", "Google", "Meta", "Amazon", "Apple", "Netflix",
+        "Stripe", "Airbnb", "Uber", "LinkedIn", "Salesforce", "Adobe",
+        "NVIDIA", "Snowflake", "Databricks",
+        "OpenAI", "Anthropic", "Cohere", "Hugging Face",
+        "Pinecone", "Weaviate", "Qdrant",
+    ],
+}
+
+# ---------------------------------------------------------------------------
+# Company tier (for company-quality feature)
+# ---------------------------------------------------------------------------
+
+COMPANY_TIER: dict[int, set[str]] = {
+    3: {
+        "Google", "Meta", "Microsoft", "Amazon", "Apple", "Netflix",
+        "Stripe", "Airbnb", "Uber", "LinkedIn",
+        "Salesforce", "Adobe", "NVIDIA", "Snowflake", "Databricks",
+        "OpenAI", "Anthropic", "Cohere", "Hugging Face",
+        "Pinecone", "Weaviate", "Qdrant",
+    },
+    2: {
+        "Razorpay", "Zerodha", "CRED", "PhonePe", "Paytm",
+        "Swiggy", "Zomato", "Flipkart", "Meesho", "Urban Company",
+        "Postman", "Freshworks", "Zoho", "Chargebee",
+        "BrowserStack", "Sprinklr", "CleverTap", "MoEngage", "ShareChat",
+        "InMobi", "Dream11", "Groww", "Cars24",
+        "Ola", "Rivigo", "Delhivery", "Udaan", "Oyo",
+        "Intel", "IBM", "Oracle", "Redrob",
+        # Indian AI-first labs
+        "Sarvam AI", "Niramai", "Rephrase", "Fractal", "Yellow.ai",
+        "Mad Street Den", "Wysa", "Haptik", "Uniphore", "Observe.AI",
+        "Turing", "Locobuzz", "Verloop", "Bobble AI", "Galytix",
+        "Locus", "Krutrim",
+        # Ed-tech / health-tech (product companies with AI teams)
+        "BYJU", "BYJUS", "Unacademy", "Vedantu", "upGrad",
+        "PolicyBazaar",
+    },
+}

redrob/data.py

@@ -0,0 +1,295 @@
+"""Candidate canonicalisation.
+
+Streams the 100k JSONL, normalises each record into a compact dict, and
+optionally writes a parquet cache for fast reloads.
+"""
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Dict, Iterator, List, Optional, Tuple
+
+import pandas as pd
+
+from . import config
+
+
+# ---------------------------------------------------------------------------
+# Canonical record
+# ---------------------------------------------------------------------------
+
+@dataclass
+class CanonicalCandidate:
+    candidate_id: str
+    name: str
+    headline: str
+    summary: str
+    location: str
+    country: str
+    yoe: float
+    current_title: str
+    current_company: str
+    current_company_size: str
+    current_industry: str
+    career: List[Dict] = field(default_factory=list)
+    edu: List[Dict] = field(default_factory=list)
+    skills: List[Dict] = field(default_factory=list)
+    certs: List[Dict] = field(default_factory=list)
+    languages: List[Dict] = field(default_factory=list)
+    signals: Dict = field(default_factory=dict)
+    # Derived
+    text_corpus: str = ""
+    career_total_months: int = 0
+    career_overlap_months: int = 0
+    career_industries: List[str] = field(default_factory=list)
+    career_titles: List[str] = field(default_factory=list)
+    career_companies: List[str] = field(default_factory=list)
+    skill_names: List[str] = field(default_factory=list)
+    skill_durations_max: Dict[str, int] = field(default_factory=dict)
+    has_ai_skill: int = 0
+
+
+# ---------------------------------------------------------------------------
+# Text tokenisation (BM25 corpus)
+# ---------------------------------------------------------------------------
+
+_TOKEN_RE = re.compile(r"[a-z0-9+#\.]+")
+# Pre-compile lowercasing noop; use str.translate for speed
+_TR_TABLE = str.maketrans("", "", ".,;:!?'\"()[]{}<>")
+_SKILL_HINT_RE = re.compile(
+    r"\b("
+    r"pytorch|tensorflow|transformers?|hugging|llm|rag|retrieval|recommend"
+    r"|ranking|ranker|search|embedding|vector|faiss|pinecone|weaviate|qdrant"
+    r"|milvus|opensearch|elasticsearch|bm25|xgboost|lightgbm|catboost"
+    r"|learning[\s-]?to[\s-]?rank|lambdarank|lora|qlora|peft|fine[\s-]?tun"
+    r"|sentence[\s-]?transformer|bge|e5|mlops|kubeflow|mlflow|triton|onnx"
+    r"|nlp|computer vision|cv|numpy|pandas|spark|airflow|kafka|databricks"
+    r"|snowflake|distributed|production|deploy|inference|eval|ndcg|mrr|map"
+    r"|a/b|ab test|hybrid|recsys|recommender|search relevance|rank model"
+    r"|promotion velocity"
+    r")\b",
+    re.IGNORECASE,
+)
+
+MAX_TEXT_CHARS = 4000  # BM25 memory control
+
+
+def _tokenize(text: str) -> List[str]:
+    """Unigram tokenization only. Bigrams removed because they reward
+    keyword-stuffing honeypot profiles that pack AI buzzwords densely."""
+    if not text:
+        return []
+    text = text.lower().translate(_TR_TABLE)
+    return _TOKEN_RE.findall(text)
+
+
+def _build_text_corpus(c: Dict) -> str:
+    parts: List[str] = []
+    if c.get("headline"):
+        parts.append(c["headline"])
+    if c.get("summary"):
+        parts.append(c["summary"])
+    for ch in c.get("career_history", []):
+        if ch.get("title"):
+            parts.append(ch["title"])
+        if ch.get("company"):
+            parts.append(ch["company"])
+        if ch.get("description"):
+            parts.append(ch["description"])
+    for s in c.get("skills", []):
+        parts.append(s.get("name", ""))
+    for cert in c.get("certifications", []):
+        parts.append(cert.get("name", ""))
+    for lang in c.get("languages", []):
+        parts.append(lang.get("language", ""))
+    for edu in c.get("education", []):
+        parts.append(edu.get("degree", ""))
+        parts.append(edu.get("field_of_study", ""))
+    text = " \n ".join([p for p in parts if p])
+    return text[:MAX_TEXT_CHARS]
+
+
+# ---------------------------------------------------------------------------
+# Career timeline integrity
+# ---------------------------------------------------------------------------
+
+def _months_overlap(a_start: str, a_end: Optional[str], b_start: str, b_end: Optional[str]) -> int:
+    """Approx months overlap of two (start,end) date pairs in YYYY-MM-DD."""
+    if not a_start or not b_start:
+        return 0
+    try:
+        a1 = _parse_date(a_start)
+        a2 = _parse_date(a_end) if a_end else _TODAY
+        b1 = _parse_date(b_start)
+        b2 = _parse_date(b_end) if b_end else _TODAY
+    except Exception:
+        return 0
+    if a1 > a2 or b1 > b2:
+        return 0
+    start = a1 if a1 > b1 else b1
+    end = a2 if a2 < b2 else b2
+    if end < start:
+        return 0
+    return max(0, int((end - start) // 30))
+
+
+# Pre-parse today's date once.
+_TODAY_DT = __import__("datetime").datetime.now()
+_TODAY = _TODAY_DT.year * 12 + _TODAY_DT.month
+
+
+def _parse_date(s: str) -> int:
+    """Return YYYY*12+MM for a YYYY-MM-DD string. Very fast path."""
+    if not s or len(s) < 7:
+        return 0
+    try:
+        y = int(s[0:4])
+        m = int(s[5:7])
+        if not (1970 <= y <= 2099 and 1 <= m <= 12):
+            return 0
+        return y * 12 + m
+    except Exception:
+        return 0
+
+
+def _career_timeline(career: List[Dict]) -> Tuple[int, int, int, int]:
+    """Return (sum_months, overlap_months, has_overlap, is_impossible)."""
+    total = 0
+    overlap = 0
+    n_impossible = 0
+    n = len(career)
+    spans = []
+    for ch in career:
+        dur = int(ch.get("duration_months") or 0)
+        if dur < 0 or dur > 600:
+            n_impossible += 1
+        total += dur
+        a1 = _parse_date(ch.get("start_date", ""))
+        a2 = _parse_date(ch.get("end_date")) if ch.get("end_date") else _TODAY
+        is_cur = bool(ch.get("is_current"))
+        spans.append((a1, a2, is_cur))
+    # pairwise overlap of roles where at least one is not "current"
+    for i in range(n):
+        a1, a2, ac = spans[i]
+        if a1 == 0 or a2 == 0:
+            continue
+        for j in range(i + 1, n):
+            b1, b2, bc = spans[j]
+            if b1 == 0 or b2 == 0:
+                continue
+            if ac and bc:
+                continue
+            start = a1 if a1 > b1 else b1
+            end = a2 if a2 < b2 else b2
+            if end > start and (end - start) >= 6:
+                overlap += (end - start)
+    return total, overlap, int(overlap >= 6), n_impossible
+
+
+# ---------------------------------------------------------------------------
+# Canonicalisation
+# ---------------------------------------------------------------------------
+
+def canonicalise(c: Dict) -> CanonicalCandidate:
+    p = c.get("profile", {}) or {}
+    career = c.get("career_history", []) or []
+    skills = c.get("skills", []) or []
+    sig = c.get("redrob_signals", {}) or {}
+
+    cc = CanonicalCandidate(
+        candidate_id=c.get("candidate_id", ""),
+        name=p.get("anonymized_name", ""),
+        headline=p.get("headline", ""),
+        summary=p.get("summary", ""),
+        location=p.get("location", ""),
+        country=p.get("country", ""),
+        yoe=float(p.get("years_of_experience") or 0.0),
+        current_title=p.get("current_title", ""),
+        current_company=p.get("current_company", ""),
+        current_company_size=p.get("current_company_size", ""),
+        current_industry=p.get("current_industry", ""),
+        career=career,
+        edu=c.get("education", []) or [],
+        skills=skills,
+        certs=c.get("certifications", []) or [],
+        languages=c.get("languages", []) or [],
+        signals=sig,
+    )
+
+    # Text corpus
+    cc.text_corpus = _build_text_corpus(c)
+
+    # Career timeline metrics
+    total, overlap, _has_overlap, _imp = _career_timeline(career)
+    cc.career_total_months = total
+    cc.career_overlap_months = overlap
+    cc.career_industries = list({(ch.get("industry") or "").strip() for ch in career if ch.get("industry")})
+    cc.career_titles = [(ch.get("title") or "").strip() for ch in career if ch.get("title")]
+    cc.career_companies = [(ch.get("company") or "").strip() for ch in career if ch.get("company")]
+
+    # Skills
+    cc.skill_names = [s.get("name", "").strip() for s in skills if s.get("name")]
+    for s in skills:
+        n = (s.get("name") or "").strip()
+        d = int(s.get("duration_months") or 0)
+        if n:
+            cc.skill_durations_max[n] = max(cc.skill_durations_max.get(n, 0), d)
+    # has_ai_skill flag
+    skills_lower = {n.lower() for n in cc.skill_names}
+    if skills_lower & {t.lower() for t in config.STRUCTURED_GATE_TERMS}:
+        cc.has_ai_skill = 1
+    if _SKILL_HINT_RE.search(cc.text_corpus):
+        cc.has_ai_skill = 1
+
+    return cc
+
+
+# ---------------------------------------------------------------------------
+# Streaming loader
+# ---------------------------------------------------------------------------
+
+def iter_canonical(path: Optional[Path] = None) -> Iterator[CanonicalCandidate]:
+    """Yield canonical records from a JSONL file.
+
+    Reads line-by-line. The 100k file is ~487MB; line-based parsing with
+    json.loads is ~4x faster than ijson on this file because ijson chokes
+    on a small fraction of records with non-standard whitespace, and the
+    total memory is fine since we only hold one record at a time.
+    """
+    p = Path(path) if path else config.DEFAULT_CANDIDATES_JSONL
+    if not p.exists():
+        raise FileNotFoundError(f"candidates file not found: {p}")
+
+    with open(p, "r", encoding="utf-8", errors="replace") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                yield canonicalise(json.loads(line))
+            except json.JSONDecodeError:
+                # skip malformed records
+                continue
+
+
+def load_or_build_parquet(path: Optional[Path] = None) -> pd.DataFrame:
+    """Return a pandas DataFrame of canonical candidates, building the
+    parquet cache on first call."""
+    p = Path(path) if path else config.DEFAULT_CANDIDATES_JSONL
+    if config.CANDIDATES_PARQUET.exists():
+        return pd.read_parquet(config.CANDIDATES_PARQUET)
+
+    rows: List[Dict] = []
+    for cc in iter_canonical(p):
+        rows.append(asdict(cc))
+    df = pd.DataFrame(rows)
+    df.to_parquet(config.CANDIDATES_PARQUET, index=False)
+    return df
+
+
+if __name__ == "__main__":
+    df = load_or_build_parquet()
+    print(f"loaded {len(df):,} candidates -> {config.CANDIDATES_PARQUET}")
+    print("columns:", list(df.columns)[:20])

redrob/features/__init__.py

@@ -0,0 +1,13 @@
+"""Feature engineering: title fit, behavioral recruitability, honeypot detection."""
+from .title_features import title_fit_features, company_tier_features
+from .behavioral import recruitability_features
+from .honeypot import honeypot_features
+from .skill_features import skill_features
+
+__all__ = [
+    "title_fit_features",
+    "company_tier_features",
+    "recruitability_features",
+    "honeypot_features",
+    "skill_features",
+]

redrob/features/behavioral.py

@@ -0,0 +1,111 @@
+"""Behavioral recruitability features from the 23 Redrob signals."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+import numpy as np
+import pandas as pd
+
+
+def _to_dt(s):
+    if not s:
+        return None
+    try:
+        return datetime.strptime(str(s)[:10], "%Y-%m-%d")
+    except Exception:
+        return None
+
+
+def recruitability_features(df: pd.DataFrame) -> pd.DataFrame:
+    """Compute a 0..1 recruitability composite plus sub-features."""
+    n = len(df)
+    sig_col = "signals"
+    if sig_col not in df.columns:
+        return pd.DataFrame({
+            "recruit_open_to_work": np.zeros(n),
+            "recruit_response_rate": np.zeros(n),
+            "recruit_verified": np.zeros(n),
+            "recruit_completeness": np.zeros(n),
+            "recruit_recency": np.zeros(n),
+            "recruit_notice_ok": np.zeros(n),
+            "recruit_recruiter_saves": np.zeros(n),
+            "recruit_interview_completion": np.zeros(n),
+            "recruit_offer_acceptance": np.zeros(n),
+            "recruit_github": np.zeros(n),
+            "recruitability": np.zeros(n),
+        })
+
+    def get(row, key, default=0):
+        d = row.get(sig_col)
+        if d is None:
+            d = {}
+        elif hasattr(d, "tolist"):
+            d = d.tolist() if d else {}
+        if not isinstance(d, dict):
+            return default
+        v = d.get(key, default)
+        return v if v is not None else default
+
+    open_to_work = np.array([float(get(r, "open_to_work_flag", 0)) for _, r in df.iterrows()])
+    response_rate = np.array([float(get(r, "recruiter_response_rate", 0.0)) for _, r in df.iterrows()])
+    verified_email = np.array([float(get(r, "verified_email", 0)) for _, r in df.iterrows()])
+    verified_phone = np.array([float(get(r, "verified_phone", 0)) for _, r in df.iterrows()])
+    linkedin = np.array([float(get(r, "linkedin_connected", 0)) for _, r in df.iterrows()])
+    completeness = np.array([float(get(r, "profile_completeness_score", 0.0)) for _, r in df.iterrows()]) / 100.0
+    notice = np.array([float(get(r, "notice_period_days", 90)) for _, r in df.iterrows()])
+    saves = np.array([float(get(r, "saved_by_recruiters_30d", 0)) for _, r in df.iterrows()])
+    interview_completion = np.array([float(get(r, "interview_completion_rate", 0.0)) for _, r in df.iterrows()])
+    offer_acceptance = np.array([float(get(r, "offer_acceptance_rate", -1.0)) for _, r in df.iterrows()])
+    github = np.array([float(get(r, "github_activity_score", -1.0)) for _, r in df.iterrows()])
+
+    # Recency: anchored to fixed reference date for determinism.
+    # Reference = 2026-06-01 (near dataset creation date).
+    # 90-day half-life exponential decay.
+    from datetime import datetime as _dt
+    _REF_DATE = _dt(2026, 6, 1)
+    recency = np.zeros(n)
+    for i, (_, r) in enumerate(df.iterrows()):
+        d = _to_dt(get(r, "last_active_date", None))
+        if d is None:
+            recency[i] = 0.0
+            continue
+        days = (_REF_DATE - d).days
+        import math
+        recency[i] = max(0.0, math.exp(-max(0, days) / 90.0))
+
+    # Notice period
+    notice_ok = np.where(notice <= 30, 1.0, np.maximum(0.0, 1.0 - (notice - 30) / 150.0))
+    recruiter_saves = np.minimum(saves / 20.0, 1.0)
+    offer_acc = np.where(offer_acceptance < 0, 0.3, offer_acceptance)
+    offer_acc = np.clip(offer_acc, 0.0, 1.0)
+    github_norm = np.where(github < 0, 0.0, github / 100.0)
+    verified = (verified_email + verified_phone + linkedin) / 3.0
+
+    composite = (
+        0.18 * open_to_work
+        + 0.10 * verified
+        + 0.10 * response_rate
+        + 0.06 * interview_completion
+        + 0.06 * offer_acc
+        + 0.10 * completeness
+        + 0.08 * recency
+        + 0.10 * notice_ok
+        + 0.10 * recruiter_saves
+        + 0.06 * github_norm
+        + 0.06 * open_to_work * verified  # bonus: open AND verified
+    )
+    composite = np.clip(composite, 0.0, 1.0)
+
+    return pd.DataFrame({
+        "recruit_open_to_work": open_to_work,
+        "recruit_response_rate": response_rate,
+        "recruit_verified": verified,
+        "recruit_completeness": completeness,
+        "recruit_recency": recency,
+        "recruit_notice_ok": notice_ok,
+        "recruit_recruiter_saves": recruiter_saves,
+        "recruit_interview_completion": interview_completion,
+        "recruit_offer_acceptance": offer_acc,
+        "recruit_github": github_norm,
+        "recruitability": composite,
+    })

redrob/features/honeypot.py

@@ -0,0 +1,253 @@
+"""Explicit honeypot detection.
+
+We do not learn the contradictions. We encode ten hard rules and
+combine them into a single penalty ∈ [0, 1].
+"""
+from __future__ import annotations
+
+import re
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+from .. import config
+
+
+_SKILL_AI = re.compile(
+    r"\b(pytorch|tensorflow|transformers?|hugging|llm|rag|retrieval|recommend"
+    r"|ranking|ranker|search|embedding|vector|faiss|pinecone|weaviate|qdrant"
+    r"|milvus|opensearch|elasticsearch|bm25|xgboost|lightgbm|catboost"
+    r"|learning[\s-]?to[\s-]?rank|lambdarank|lora|qlora|peft|fine[\s-]?tun"
+    r"|sentence[\s-]?transformer|bge|e5|mlops|kubeflow|mlflow|triton|onnx)\b",
+    re.IGNORECASE,
+)
+
+
+def _is_ai_skill(name: str) -> bool:
+    return bool(_SKILL_AI.search(name or ""))
+
+
+def honeypot_features(df: pd.DataFrame) -> pd.DataFrame:
+    n = len(df)
+    p = np.zeros(n)
+    rule_flags = {f"hp_rule_{i}": np.zeros(n) for i in range(10)}
+
+    # Pre-extract per-row arrays
+    skill_names = []
+    for x in df["skill_names"].tolist():
+        if x is None:
+            skill_names.append([])
+        elif hasattr(x, "tolist"):
+            skill_names.append(x.tolist())
+        else:
+            skill_names.append(x)
+    skills_full = []
+    for x in df["skills"].tolist():
+        if x is None:
+            skills_full.append([])
+        elif hasattr(x, "tolist"):
+            skills_full.append(x.tolist())
+        else:
+            skills_full.append(x)
+    yoe = df["yoe"].astype(float).to_numpy()
+    career_total = df["career_total_months"].astype(float).to_numpy() / 12.0
+    overlap = df["career_overlap_months"].astype(float).to_numpy()
+    edu = []
+    for x in df["edu"].tolist():
+        if x is None:
+            edu.append([])
+        elif hasattr(x, "tolist"):
+            edu.append(x.tolist())
+        else:
+            edu.append(x)
+    sigs = []
+    for s in df["signals"].tolist():
+        if s is None:
+            sigs.append({})
+        elif hasattr(s, "tolist"):
+            sigs.append(s.tolist() if s else {})
+        else:
+            sigs.append(s)
+    profile_completeness = np.array([
+        (s or {}).get("profile_completeness_score", 50.0) if isinstance(s, dict) else 50.0
+        for s in sigs
+    ])
+    salary_min = np.array([
+        ((s or {}).get("expected_salary_range_inr_lpa") or {}).get("min", 0.0) if isinstance(s, dict) else 0.0
+        for s in sigs
+    ])
+    salary_max = np.array([
+        ((s or {}).get("expected_salary_range_inr_lpa") or {}).get("max", 0.0) if isinstance(s, dict) else 0.0
+        for s in sigs
+    ])
+    search_appearance = np.array([
+        (s or {}).get("search_appearance_30d", 0) if isinstance(s, dict) else 0
+        for s in sigs
+    ])
+    saved_by_recruiters = np.array([
+        (s or {}).get("saved_by_recruiters_30d", 0) if isinstance(s, dict) else 0
+        for s in sigs
+    ])
+
+    for i in range(n):
+        contrib = 0.0
+        _skf = skills_full[i]
+        if _skf is None:
+            _skf = []
+        elif hasattr(_skf, "tolist"):
+            _skf = _skf.tolist()
+        _sn = skill_names[i]
+        if _sn is None:
+            _sn = []
+        elif hasattr(_sn, "tolist"):
+            _sn = _sn.tolist()
+
+        # Rule 1: expert + duration_months < 6 (capped)
+        cnt = 0
+        for s in _skf:
+            if s.get("proficiency") == "expert" and (s.get("duration_months") or 0) < 6 and _is_ai_skill(s.get("name", "")):
+                cnt += 1
+        rule_flags["hp_rule_0"][i] = min(cnt * 0.25, 0.5)
+        contrib += rule_flags["hp_rule_0"][i]
+
+        # Rule 2: advanced/expert + 0 months
+        cnt2 = 0
+        for s in _skf:
+            if s.get("proficiency") in ("advanced", "expert") and (s.get("duration_months") or 0) == 0 and _is_ai_skill(s.get("name", "")):
+                cnt2 += 1
+        rule_flags["hp_rule_1"][i] = min(cnt2 * 0.15, 0.3)
+        contrib += rule_flags["hp_rule_1"][i]
+
+        # Rule 3: career duration > YOE + 1.5 years
+        if yoe[i] > 0 and career_total[i] > yoe[i] + 1.5:
+            rule_flags["hp_rule_2"][i] = 0.25
+            contrib += 0.25
+
+        # Rule 4: concurrent overlap > 12 months
+        if overlap[i] >= 12:
+            rule_flags["hp_rule_3"][i] = 0.20
+            contrib += 0.20
+
+        # Rule 5: education anomalies
+        bad_edu = 0
+        _edu = edu[i]
+        if _edu is None:
+            _edu = []
+        elif hasattr(_edu, "tolist"):
+            _edu = _edu.tolist()
+        for e in _edu:
+            sy = int(e.get("start_year") or 0)
+            ey = int(e.get("end_year") or 0)
+            if ey and sy and ey < sy:
+                bad_edu += 1
+            elif sy and ey:
+                deg = (e.get("degree") or "").lower()
+                expected = 2 if "phd" in deg or "doctor" in deg else (1 if "master" in deg or "m.sc" in deg or "m.s." in deg or "m.e" in deg or "m.tech" in deg or "mba" in deg else 4)
+                if (ey - sy) < expected - 1 and ey >= 2020:
+                    bad_edu += 1
+        if bad_edu:
+            rule_flags["hp_rule_4"][i] = min(bad_edu * 0.10, 0.20)
+            contrib += rule_flags["hp_rule_4"][i]
+
+        # Rule 6: many advanced skills with low YOE
+        adv = sum(1 for s in _skf if s.get("proficiency") in ("advanced", "expert"))
+        if adv >= 6 and yoe[i] < 3:
+            rule_flags["hp_rule_5"][i] = 0.25
+            contrib += 0.25
+
+        # Rule 7 v2: title-skill contradiction (loosened).
+        # Fires when current_title is non-eng AND
+        #   (a) every historical title is non-eng OR
+        #   (b) max assessment >= 80 (unrealistic for non-eng) OR
+        #   (c) career_total_months < yoe * 12 * 0.85 (implausibly short career).
+        if adv >= 5:
+            non_eng_kw = (
+                "marketing", "hr ", "accountant", "sales ", "graphic", "content writer",
+                "civil", "mechanical", "customer support", "operations manager",
+                "project manager", "business analyst", "qa ", "frontend", "java developer",
+                ".net", "mobile developer", "devops", "cloud engineer", "data analyst",
+            )
+            cur = (df.iloc[i]["current_title"] or "").lower()
+            hist_titles = df.iloc[i]["career_titles"]
+            if hist_titles is None:
+                hist_titles = []
+            elif hasattr(hist_titles, "tolist"):
+                hist_titles = hist_titles.tolist()
+            non_eng_cur = any(k in cur for k in non_eng_kw)
+            all_non_eng_hist = all(any(k in (t or "").lower() for k in non_eng_kw)
+                                    for t in hist_titles if t)
+            # Compute max assessment score
+            max_assess = 0
+            sig = sigs[i]
+            if isinstance(sig, dict):
+                sas = sig.get("skill_assessment_scores") or {}
+                for v in (sas or {}).values():
+                    if isinstance(v, (int, float)) and v > max_assess:
+                        max_assess = v
+            # Career-impossible
+            impossible_career = (
+                yoe[i] > 0 and career_total[i] < yoe[i] * 0.85
+            )
+            if non_eng_cur and (all_non_eng_hist or max_assess >= 80 or impossible_career):
+                rule_flags["hp_rule_6"][i] = 0.30
+                contrib += 0.30
+
+        # Rule 8: profile stub with rich skills
+        if profile_completeness[i] < 35 and adv >= 8:
+            rule_flags["hp_rule_7"][i] = 0.15
+            contrib += 0.15
+
+        # Rule 9: salary inversion
+        if salary_min[i] > salary_max[i] and salary_min[i] > 0:
+            rule_flags["hp_rule_8"][i] = 0.10
+            contrib += 0.10
+
+        # Rule 10: signals fakery
+        if search_appearance[i] > 500 and saved_by_recruiters[i] == 0:
+            rule_flags["hp_rule_9"][i] = 0.10
+            contrib += 0.10
+
+        p[i] = min(contrib, 1.0)
+
+    out = pd.DataFrame({"honeypot_penalty": p})
+
+    # Rule 11: behavioral twins (post-pass) — detect candidates whose
+    # (name, current_title, round(yoe,1)) match another candidate and
+    # penalise the weaker-behavior twin. Matches the spec's explicit
+    # "behavioral twins" trap (530 twin-keys, 1,061 candidates).
+    from collections import defaultdict
+    sigs_for_twins = sigs
+    rr_arr = np.array([
+        (s.get("recruiter_response_rate", 0.0) if isinstance(s, dict) else 0.0)
+        for s in sigs_for_twins
+    ], dtype=np.float32)
+    recency_arr = np.array([
+        (s.get("recency_score", s.get("last_active_date_score", 0.5)) if isinstance(s, dict) else 0.5)
+        for s in sigs_for_twins
+    ], dtype=np.float32)
+    names = df["name"].fillna("").astype(str).tolist() if "name" in df.columns \
+            else df["candidate_id"].astype(str).tolist()
+    titles = df["current_title"].fillna("").astype(str).tolist()
+    yoe_round = np.round(yoe, 1)
+    twin_groups: dict = defaultdict(list)
+    for i in range(n):
+        key = (names[i].strip().lower(), titles[i].strip().lower(), float(yoe_round[i]))
+        if key[0] == "":
+            continue
+        twin_groups[key].append(i)
+    twin_penalty = np.zeros(n, dtype=np.float32)
+    for key, idxs in twin_groups.items():
+        if len(idxs) < 2:
+            continue
+        # Pick the strongest twin (highest rr + recency); penalise the rest
+        strengths = rr_arr[idxs] + recency_arr[idxs]
+        order = np.argsort(-strengths)
+        leader = idxs[order[0]]
+        for weaker in [idxs[i] for i in order[1:]]:
+            twin_penalty[weaker] += 0.15
+    out["honeypot_penalty"] = np.minimum(out["honeypot_penalty"].to_numpy() + twin_penalty, 1.0)
+    out["hp_rule_10"] = twin_penalty
+    for k, v in rule_flags.items():
+        out[k] = v
+    return out

redrob/features/skill_features.py

@@ -0,0 +1,237 @@
+"""Skill-related features: must-have coverage, assessments, durations."""
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from .. import config
+
+
+_CORE = {s.lower() for s in config.CORE_COMPETENCIES}
+_ADJ = {s.lower() for s in config.ADJACENT_COMPETENCIES}
+_NEG = {s.lower() for s in config.NEGATIVE_COMPETENCIES}
+_CRIT = {s.lower() for s in config.JD_CRITICAL}
+_NICE = {s.lower() for s in config.JD_NICE_TO_HAVE}
+_SKILL_WEIGHT = {s: 2.0 for s in _CRIT}
+_SKILL_WEIGHT.update({s: 1.0 for s in _NICE})
+for s in _CORE - _CRIT - _NICE:
+    _SKILL_WEIGHT.setdefault(s, 0.5)
+# Max possible weighted score (6 critical + 4 nice-to-have)
+_MAX_WEIGHT = sum(sorted(_SKILL_WEIGHT.values(), reverse=True)[:10])
+
+
+# ----------------------------------------------------------------------
+# Skill alias / canonicalization map (added by agent review, Exp #8)
+# ----------------------------------------------------------------------
+# Maps variant skill names (as they appear in candidates.jsonl and
+# Redrob assessments) to their canonical JD-core counterparts.
+# This boosts recall for candidates who list the same skill under a
+# slightly different name (e.g. "Fine-tuning LLMs" → "Fine-tuning",
+# "pgvector" → vector DB family, "Search Relevance" → BM25 family).
+_SKILL_ALIAS = {
+    # Retrieval / vector DB family
+    "pgvector": "vector database",
+    "pgvector (postgres vector)": "vector database",
+    "vector database": "vector database",
+    "vector databases": "vector database",
+    "vector db": "vector database",
+    "dense retrieval": "dense retrieval",
+    "dense passage retrieval": "dense retrieval",
+    "dense passage retrieval (dpr)": "dense retrieval",
+    "approximate nearest neighbor": "vector database",
+    "faiss": "faiss",
+    "ann": "faiss",
+    "hnsw": "faiss",
+    "scaNN": "faiss",
+    "qdrant": "qdrant",
+    # Search ranking family
+    "search relevance": "bm25",
+    "search ranking": "bm25",
+    "relevance ranking": "learning to rank",
+    "relevance": "bm25",
+    "search engineer": "bm25",
+    "lexical search": "bm25",
+    "tf-idf": "bm25",
+    "tfidf": "bm25",
+    # LLM / fine-tuning family
+    "fine-tuning llms": "fine-tuning",
+    "llm fine-tuning": "fine-tuning",
+    "llm finetuning": "fine-tuning",
+    "fine-tuning": "fine-tuning",
+    "finetuning": "fine-tuning",
+    "instruction tuning": "fine-tuning",
+    "rlhf": "fine-tuning",
+    "rlhf (reinforcement learning from human feedback)": "fine-tuning",
+    "prompt tuning": "prompt engineering",
+    "prompt engineering": "prompt engineering",
+    "prompt design": "prompt engineering",
+    "prompt crafting": "prompt engineering",
+    "rag (retrieval-augmented generation)": "rag",
+    "retrieval augmented generation (rag)": "rag",
+    "retrieval-augmented generation (rag)": "rag",
+    "retrieval augmented generation": "rag",
+    "rag": "rag",
+    # Evaluation family
+    "learning-to-rank": "learning to rank",
+    "learning to rank (ltr)": "learning to rank",
+    "ltr (learning to rank)": "learning to rank",
+    "learning2rank": "learning to rank",
+    "lambdarank": "lambdarank",
+    "lambda rank": "lambdarank",
+    "lambdamart": "lambdarank",
+    "ndcg (normalised discounted cumulative gain)": "ndcg",
+    "mrr (mean reciprocal rank)": "mrr",
+    "map (mean average precision)": "map",
+    "mean reciprocal rank": "mrr",
+    "mean average precision": "map",
+    "normalised discounted cumulative gain": "ndcg",
+    # Embeddings / encoders
+    "sentence-transformer": "sentence transformers",
+    "sentence transformer": "sentence transformers",
+    "sentence-transformers": "sentence transformers",
+    "sbert": "sentence transformers",
+    "use (universal sentence encoder)": "sentence transformers",
+    "bge (bge embedding)": "bge",
+    "e5 (text embeddings)": "e5",
+    # Hybrid search
+    "hybrid retrieval": "hybrid search",
+    "hybrid ranking": "hybrid search",
+    "hybrid information retrieval": "hybrid search",
+    # OpenSearch / Elasticsearch
+    "elastic search": "elasticsearch",
+    "opensearch (managed elasticsearch)": "opensearch",
+    # Pinecone / Weaviate
+    "pinecone (vector database)": "pinecone",
+    "weaviate (vector database)": "weaviate",
+    # LLM
+    "large language models": "llm",
+    "large language model (llm)": "llm",
+    "llm (large language models)": "llm",
+    "llms": "llm",
+    # NLP
+    "natural language processing": "nlp",
+    "nlp (natural language processing)": "nlp",
+    # Vector Search alternative
+    "vector search (semantic search)": "embeddings",
+    "vector search": "embeddings",
+    "semantic search": "embeddings",
+    "approximate nearest neighbours": "vector database",
+}
+
+
+def _canon_skill(name: str) -> str:
+    """Return the canonical form of a skill name. Falls back to lower(name)."""
+    n = (name or "").strip().lower()
+    if not n:
+        return ""
+    return _SKILL_ALIAS.get(n, n)
+
+
+def _canon_skill_set(names: set) -> set:
+    """Apply canonicalization to a set of skill names."""
+    return {_canon_skill(n) for n in names if n}
+
+
+def skill_features(df: pd.DataFrame) -> pd.DataFrame:
+    n = len(df)
+
+    def _as_list(v):
+        if v is None:
+            return []
+        if hasattr(v, "tolist"):
+            return v.tolist()
+        return v
+
+    skills_full_col = [_as_list(x) for x in df["skills"].tolist()]
+    sigs_col = [_as_list(x) for x in df["signals"].tolist()]
+    n_core_hits = np.zeros(n, dtype=np.int32)
+    n_adj_hits = np.zeros(n, dtype=np.int32)
+    n_neg_hits = np.zeros(n, dtype=np.int32)
+    n_advanced = np.zeros(n, dtype=np.int32)
+    n_expert = np.zeros(n, dtype=np.int32)
+    assessment_max = np.zeros(n, dtype=np.float32)
+    assessment_mean = np.zeros(n, dtype=np.float32)
+    endorsement_log_mean = np.zeros(n, dtype=np.float32)
+    duration_log_mean = np.zeros(n, dtype=np.float32)
+    skill_count = np.zeros(n, dtype=np.int32)
+    jaccard_core = np.zeros(n, dtype=np.float32)
+    jd_crit_score = np.zeros(n, dtype=np.float32)
+    # Alias-canonicalized sets (for matching against core/jd-critical)
+    names_canon_lc_col = []
+
+    for i in range(n):
+        skills_full = skills_full_col[i]
+        if skills_full is None:
+            skills_full = []
+        names = [s.get("name", "").strip() for s in skills_full if s.get("name")]
+        names_lc = {n_.lower() for n_ in names if n_}
+        names_canon_lc = _canon_skill_set(names_lc)
+        names_canon_lc_col.append(names_canon_lc)
+        skill_count[i] = len(names)
+        n_core_hits[i] = len(names_canon_lc & _CORE)
+        n_adj_hits[i] = len(names_canon_lc & _ADJ)
+        n_neg_hits[i] = len(names_canon_lc & _NEG)
+        n_advanced[i] = sum(1 for s in skills_full if s.get("proficiency") in ("advanced",))
+        n_expert[i] = sum(1 for s in skills_full if s.get("proficiency") == "expert")
+
+        # JD-criticality weighted score (uses canonical names)
+        w_sum = sum(_SKILL_WEIGHT.get(n, 0.0) for n in names_canon_lc)
+        jd_crit_score[i] = min(w_sum / max(_MAX_WEIGHT, 1e-9), 1.0)
+
+        # Assessments — canonicalize keys to merge aliases (e.g. "pgvector" and
+        # "vector database" both map to the same canonical key for aggregation)
+        sig = sigs_col[i]
+        if sig is None:
+            sig = {}
+        if isinstance(sig, dict):
+            sas = sig.get("skill_assessment_scores") or {}
+            # Canonicalize keys and merge values (take max for same canonical)
+            canon_sas: dict = {}
+            for k, v in sas.items():
+                if v is None or not isinstance(v, (int, float)):
+                    continue
+                ck = _canon_skill(k)
+                if ck in canon_sas:
+                    if v > canon_sas[ck]:
+                        canon_sas[ck] = float(v)
+                else:
+                    canon_sas[ck] = float(v)
+            if canon_sas:
+                vals = list(canon_sas.values())
+                assessment_max[i] = max(vals)
+                assessment_mean[i] = sum(vals) / len(vals)
+        # Endorsement / duration log means
+        e_vals = [int(s.get("endorsements") or 0) for s in skills_full]
+        d_vals = [int(s.get("duration_months") or 0) for s in skills_full]
+        if e_vals:
+            endorsement_log_mean[i] = float(np.mean(np.log1p(e_vals)))
+        if d_vals:
+            duration_log_mean[i] = float(np.mean(np.log1p(d_vals)))
+
+        # Jaccard of candidate's skills with core (canonicalized)
+        if names_canon_lc and _CORE:
+            inter = len(names_canon_lc & _CORE)
+            union = len(names_canon_lc | _CORE)
+            jaccard_core[i] = inter / max(union, 1)
+
+    must_have_coverage = np.minimum(n_core_hits / 8.0, 1.0)  # saturates at 8 core skills
+    adj_coverage = np.minimum(n_adj_hits / 5.0, 1.0)
+    ai_signal_strength = (n_core_hits + 0.5 * n_adj_hits) / np.maximum(1, skill_count)
+
+    return pd.DataFrame({
+        "skill_count": skill_count,
+        "n_core_skills": n_core_hits,
+        "n_adj_skills": n_adj_hits,
+        "n_neg_skills": n_neg_hits,
+        "n_advanced_skills": n_advanced,
+        "n_expert_skills": n_expert,
+        "must_have_coverage": must_have_coverage,
+        "adjacent_coverage": adj_coverage,
+        "ai_signal_strength": ai_signal_strength,
+        "assessment_max": assessment_max,
+        "assessment_mean": assessment_mean,
+        "endorsement_log_mean": endorsement_log_mean,
+        "duration_log_mean": duration_log_mean,
+        "jaccard_core": jaccard_core,
+        "jd_criticality_score": jd_crit_score,
+    })

redrob/features/title_features.py

@@ -0,0 +1,148 @@
+"""Graduated title-fit scoring.
+
+Combines:
+  - exact role-group match on current and historical titles
+  - embedding cosine of current title to the centroid of target titles
+  - promotion velocity / role progression features
+"""
+from __future__ import annotations
+
+import re
+from typing import List
+
+import numpy as np
+import pandas as pd
+
+from .. import config
+
+
+def _normalise(title: str) -> str:
+    t = (title or "").lower()
+    t = re.sub(r"[^a-z0-9+\-# ]+", " ", t)
+    t = re.sub(r"\s+", " ", t).strip()
+    return t
+
+
+def _title_group(title_norm: str) -> str | None:
+    best = None
+    for group, members in config.TITLE_ROLE_GROUPS.items():
+        for m in members:
+            if m in title_norm:
+                if best is None or config.TITLE_GROUP_WEIGHT[group] > config.TITLE_GROUP_WEIGHT[best]:
+                    best = group
+    return best
+
+
+def title_fit_features(df: pd.DataFrame) -> pd.DataFrame:
+    """Return a DataFrame of per-candidate title features."""
+    current = df["current_title"].fillna("").tolist()
+    histories = df["career_titles"].tolist()
+
+    # Pre-compute best historical group across all rows
+    current_groups = [_title_group(_normalise(t)) for t in current]
+    history_groups: List[List[str]] = []
+    history_max_weight: List[float] = []
+    for hist in histories:
+        if hist is None:
+            hist = []
+        if hasattr(hist, "tolist"):
+            hist = hist.tolist()
+        gs = [_title_group(_normalise(t)) for t in (hist or [])]
+        gs = [g for g in gs if g is not None]
+        history_groups.append(gs)
+        if gs:
+            history_max_weight.append(max(config.TITLE_GROUP_WEIGHT[g] for g in gs))
+        else:
+            history_max_weight.append(0.0)
+
+    # Per-row title features
+    current_weight = [config.TITLE_GROUP_WEIGHT.get(g, 0.05) for g in current_groups]
+    is_target = [1.0 if (g in {"applied_ml", "retrieval_ranking", "nlp_llm"}) else 0.0 for g in current_groups]
+    is_services_or_noneng = [1.0 if g in {"non_target"} else 0.0 for g in current_groups]
+    is_data_platform = [1.0 if g in {"data_platform"} else 0.0 for g in current_groups]
+    is_data_science = [1.0 if g in {"data_science"} else 0.0 for g in current_groups]
+    is_generic_swe = [1.0 if g in {"generic_swe"} else 0.0 for g in current_groups]
+
+    # Promotion velocity: number of distinct titles / yoe (low if many short stints)
+    n_titles = []
+    for h in histories:
+        if h is None:
+            n_titles.append(0)
+        elif hasattr(h, "tolist"):
+            n_titles.append(len(h.tolist()))
+        else:
+            n_titles.append(len(h))
+    avg_tenure_months = []
+    for ch in df["career"].tolist():
+        if ch is None:
+            avg_tenure_months.append(0.0)
+            continue
+        if hasattr(ch, "tolist"):
+            ch = ch.tolist()
+        if not ch:
+            avg_tenure_months.append(0.0)
+            continue
+        durs = [int(c.get("duration_months") or 0) for c in (ch or [])]
+        durs = [d for d in durs if d > 0]
+        if durs:
+            avg_tenure_months.append(sum(durs) / len(durs))
+        else:
+            avg_tenure_months.append(0.0)
+
+    out = pd.DataFrame({
+        "title_weight": current_weight,
+        "title_history_max_weight": history_max_weight,
+        "is_target_title": is_target,
+        "is_noneng_title": is_services_or_noneng,
+        "is_data_platform": is_data_platform,
+        "is_data_science": is_data_science,
+        "is_generic_swe": is_generic_swe,
+        "n_career_titles": n_titles,
+        "avg_tenure_months": avg_tenure_months,
+        # blended title fit (current 70%, history 30%)
+        "title_fit_blend": [
+            0.7 * w + 0.3 * hw for w, hw in zip(current_weight, history_max_weight)
+        ],
+    })
+    return out
+
+
+# ---------------------------------------------------------------------------
+# Company tier (0-3)
+# ---------------------------------------------------------------------------
+
+def _company_tier(name: str) -> int:
+    """Map company name to tier. 3=FAANG+top-AI, 2=strong product, 1=other, 0=unknown."""
+    name = (name or "").strip()
+    if not name:
+        return 0
+    nl = name.lower()
+    for tier in (3, 2):
+        for c in config.COMPANY_TIER.get(tier, set()):
+            if c.lower() in nl or nl in c.lower():
+                return tier
+    # IT services
+    for svc in config.IT_SERVICES_PURE_PLAY:
+        if svc.lower() in nl:
+            return 0
+    return 1  # unknown product/startup
+
+
+def company_tier_features(df: pd.DataFrame) -> pd.DataFrame:
+    """Company tier features: current tier, max tier across career, top-tier flags."""
+    def _tier_from_list(lst):
+        if lst is None:
+            return []
+        if hasattr(lst, "tolist"):
+            lst = lst.tolist()
+        return [_company_tier(c) for c in (lst or [])]
+
+    current_tier = [_company_tier(c) for c in df["current_company"].fillna("").tolist()]
+    max_tiers = [max(_tier_from_list(lst)) for lst in df["career_companies"].tolist()]
+
+    return pd.DataFrame({
+        "company_tier_current": current_tier,
+        "company_tier_max": max_tiers,
+        "is_top_tier_company": [1 if t >= 3 else 0 for t in max_tiers],
+        "is_product_company_v2": [1 if t >= 1 else 0 for t in max_tiers],
+    })

redrob/graph/__init__.py

@@ -0,0 +1,13 @@
+"""Heterogeneous graph: candidate-skill-title-company-industry."""
+from .build import build_graph
+from .propagate import personalised_pagerank, ppr_axes
+from .coherence import career_coherence_scores, career_evidence_score, skill_community_features
+
+__all__ = [
+    "build_graph",
+    "personalised_pagerank",
+    "ppr_axes",
+    "career_coherence_scores",
+    "career_evidence_score",
+    "skill_community_features",
+]

redrob/graph/build.py

@@ -0,0 +1,115 @@
+"""Build a heterogeneous networkx graph over the candidate pool.
+
+Vectorised: we build long-format edge DataFrames and use
+`nx.from_edgelist` to add edges in bulk. This is roughly 50x faster
+than the iterrows approach on 100k candidates.
+"""
+from __future__ import annotations
+
+import pickle
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Tuple
+
+import networkx as nx
+import numpy as np
+import pandas as pd
+
+from .. import config
+from ..data import load_or_build_parquet
+
+
+def _safe_label(prefix: str, name: str) -> str:
+    name = (name or "").strip()
+    if not name:
+        name = "unknown"
+    return f"{prefix}::{name}"
+
+
+def _explode_lists(df: pd.DataFrame, col: str, prefix: str) -> pd.DataFrame:
+    """Return a DataFrame with columns (candidate_id, neighbour)."""
+    s = df[["candidate_id", col]].copy()
+    s[col] = s[col].apply(lambda v: list(v) if v is not None else [])
+    s = s.explode(col).dropna()
+    s["neighbour"] = s[col].apply(lambda x: _safe_label(prefix, x))
+    s = s.drop(columns=[col])
+    s = s[s["neighbour"].str.split("::", n=1).str[1] != "unknown"]
+    return s
+
+
+def build_graph(force: bool = False) -> nx.Graph:
+    if not force and config.GRAPH_PICKLE.exists():
+        with open(config.GRAPH_PICKLE, "rb") as f:
+            return pickle.load(f)
+
+    df = load_or_build_parquet()
+    G = nx.Graph()
+    cand_ids = df["candidate_id"].tolist()
+    cand_nodes = [f"cand::{c}" for c in cand_ids]
+    G.add_nodes_from((n, {"kind": "candidate"}) for n in cand_nodes)
+
+    # Skill edges
+    print("building skill edges...")
+    skill_df = _explode_lists(df, "skill_names", "skill")
+    skill_edges = list(zip(skill_df["candidate_id"].map(lambda c: f"cand::{c}"),
+                           skill_df["neighbour"]))
+    G.add_edges_from(skill_edges, kind="has_skill", weight=1.0)
+    # Capture unique skill nodes
+    print(f"  skill edges: {len(skill_edges):,}")
+
+    # Title edges
+    print("building title edges...")
+    title_df = _explode_lists(df, "career_titles", "title")
+    title_edges = list(zip(title_df["candidate_id"].map(lambda c: f"cand::{c}"),
+                           title_df["neighbour"]))
+    G.add_edges_from(title_edges, kind="held_title", weight=1.0)
+    print(f"  title edges: {len(title_edges):,}")
+
+    # Company edges
+    print("building company edges...")
+    comp_df = _explode_lists(df, "career_companies", "company")
+    comp_edges = list(zip(comp_df["candidate_id"].map(lambda c: f"cand::{c}"),
+                          comp_df["neighbour"]))
+    G.add_edges_from(comp_edges, kind="worked_at", weight=1.0)
+    print(f"  company edges: {len(comp_edges):,}")
+
+    # Industry edges
+    print("building industry edges...")
+    ind_df = _explode_lists(df, "career_industries", "industry")
+    ind_edges = list(zip(ind_df["candidate_id"].map(lambda c: f"cand::{c}"),
+                         ind_df["neighbour"]))
+    G.add_edges_from(ind_edges, kind="in_industry", weight=1.0)
+    print(f"  industry edges: {len(ind_edges):,}")
+
+    # Skill-skill co-occurrence within same candidate
+    print("building skill cooccurrence edges...")
+    cand_skill: Dict[str, List[str]] = defaultdict(list)
+    for cid, sn in skill_edges:
+        cand_skill[cid].append(sn)
+    cooccur_counts: Dict[Tuple[str, str], int] = defaultdict(int)
+    for c, skills in cand_skill.items():
+        # dedupe within candidate
+        skills = list(set(skills))
+        for i in range(len(skills)):
+            for j in range(i + 1, len(skills)):
+                a, b = skills[i], skills[j]
+                if a == b:
+                    continue
+                if a > b:
+                    a, b = b, a
+                cooccur_counts[(a, b)] += 1
+    cooccur_edges = [(a, b, {"kind": "skill_cooccur", "weight": float(c)})
+                     for (a, b), c in cooccur_counts.items()]
+    G.add_edges_from(cooccur_edges)
+    print(f"  cooccurrence edges: {len(cooccur_edges):,}")
+
+    with open(config.GRAPH_PICKLE, "wb") as f:
+        pickle.dump(G, f, protocol=pickle.HIGHEST_PROTOCOL)
+    return G
+
+
+if __name__ == "__main__":
+    import sys, time
+    t = time.time()
+    G = build_graph(force=True)
+    print(f"graph built: {G.number_of_nodes():,} nodes, {G.number_of_edges():,} edges in {time.time()-t:.1f}s")

redrob/graph/coherence.py

@@ -0,0 +1,246 @@
+"""Career coherence and skill community features."""
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+from typing import Dict, List, Tuple
+
+import networkx as nx
+import numpy as np
+import pandas as pd
+
+from .. import config
+from .build import build_graph
+
+
+# --- title embeddings (cached) -------------------------------------------------
+
+def _title_centroid_embeddings() -> Dict[str, np.ndarray]:
+    """Encode every unique title once. Uses sentence-transformers if available,
+    otherwise a one-hot over known role groups.
+    """
+    if config.TITLE_EMB_NPY.exists():
+        try:
+            with open(str(config.TITLE_EMB_NPY) + ".ids.txt", "r", encoding="utf-8") as f:
+                ids = [line.strip() for line in f if line.strip()]
+            arr = np.load(config.TITLE_EMB_NPY)
+            return {ids[i]: arr[i] for i in range(len(ids))}
+        except Exception:
+            pass
+
+    from ..data import load_or_build_parquet
+    df = load_or_build_parquet()
+    titles = sorted({(t or "").strip() for t in df["career_titles"].explode().tolist() if t})
+    try:
+        from sentence_transformers import SentenceTransformer  # type: ignore
+        from ..retrieval.dense import _load_model
+        model = _load_model()
+        embs = model.encode(
+            titles,
+            batch_size=128,
+            show_progress_bar=False,
+            convert_to_numpy=True,
+            normalize_embeddings=True,
+        ).astype(np.float32)
+    except Exception:
+        # Deterministic fallback: bag-of-words over known role keywords
+        vocab: Dict[str, int] = {}
+        for t in titles:
+            for tok in re.findall(r"[a-z]+", t.lower()):
+                if tok not in vocab:
+                    vocab[tok] = len(vocab)
+        embs = np.zeros((len(titles), max(len(vocab), 1)), dtype=np.float32)
+        for i, t in enumerate(titles):
+            for tok in re.findall(r"[a-z]+", t.lower()):
+                embs[i, vocab[tok]] = 1.0
+        n = np.linalg.norm(embs, axis=1, keepdims=True) + 1e-8
+        embs = embs / n
+
+    np.save(config.TITLE_EMB_NPY, embs)
+    with open(str(config.TITLE_EMB_NPY) + ".ids.txt", "w", encoding="utf-8") as f:
+        f.write("\n".join(titles))
+    return {titles[i]: embs[i] for i in range(len(titles))}
+
+
+def career_coherence_scores(df: pd.DataFrame) -> np.ndarray:
+    """Mean cosine similarity of consecutive title embeddings for each row."""
+    embs = _title_centroid_embeddings()
+    out = np.zeros(len(df), dtype=np.float32)
+    df_reset = df.reset_index(drop=True)
+    for i in range(len(df_reset)):
+        titles = df_reset.iloc[i]["career_titles"]
+        if titles is None:
+            titles = []
+        if hasattr(titles, "tolist"):
+            titles = titles.tolist()
+        if not titles or len(titles) < 2:
+            out[i] = 0.5  # neutral default for single-title records
+            continue
+        vecs = []
+        for t in titles:
+            v = embs.get((t or "").strip())
+            if v is not None:
+                vecs.append(v)
+        if len(vecs) < 2:
+            out[i] = 0.5
+            continue
+        sims = []
+        for j in range(1, len(vecs)):
+            s = float(np.dot(vecs[j - 1], vecs[j]))
+            sims.append(s)
+        out[i] = float(np.mean(sims))
+    return out
+
+
+# --- skill community features (Louvain over skill-skill subgraph) ------------
+
+def _louvain_communities(G: nx.Graph) -> Dict[str, int]:
+    try:
+        from networkx.algorithms.community import louvain_communities  # type: ignore
+        comms = louvain_communities(G, weight="weight", resolution=1.0, seed=42)
+        out: Dict[str, int] = {}
+        for idx, c in enumerate(comms):
+            for n in c:
+                out[n] = idx
+        return out
+    except Exception:
+        # Greedy modularity fallback
+        try:
+            from networkx.algorithms.community import greedy_modularity_communities
+            comms = list(greedy_modularity_communities(G, weight="weight"))
+            out = {}
+            for idx, c in enumerate(comms):
+                for n in c:
+                    out[n] = idx
+            return out
+        except Exception:
+            return {}
+
+
+def skill_community_features(df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """Return (community_purity, skill_rarity, graph_degree) per row.
+
+    - community_purity: fraction of candidate's skills that share a community
+      with at least one JD-core skill.
+    - skill_rarity: sum of inverse-skill-frequency * jd_match over skills.
+    - graph_degree: log1p of candidate's degree in the candidate-skill subgraph.
+    """
+    G = build_graph()
+    skill_nodes = [n for n, d in G.nodes(data=True) if d.get("kind") == "skill"]
+    skill_subg = G.subgraph(skill_nodes).copy()
+    communities = _louvain_communities(skill_subg)
+
+    jd_skill_nodes = {f"skill::{t.strip()}" for t in config.CORE_COMPETENCIES}
+    jd_community_ids = {communities.get(n) for n in jd_skill_nodes if n in communities and communities.get(n) is not None}
+    jd_community_ids.discard(None)
+
+    # Skill frequency (for rarity)
+    skill_freq: Dict[str, int] = defaultdict(int)
+    for u, v, d in G.edges(data=True):
+        if d.get("kind") == "has_skill":
+            skill_freq[v] += 1
+    total_cands = max(1, len(df))
+    idf = {s: float(np.log((1 + total_cands) / (1 + f))) for s, f in skill_freq.items()}
+
+    purity = np.zeros(len(df), dtype=np.float32)
+    rarity = np.zeros(len(df), dtype=np.float32)
+    degree = np.zeros(len(df), dtype=np.float32)
+
+    df_reset = df.reset_index(drop=True)
+    for i, row in df_reset.iterrows():
+        skill_names = row["skill_names"]
+        if skill_names is None:
+            skill_names = []
+        if hasattr(skill_names, "tolist"):
+            skill_names = skill_names.tolist()
+        skills = [f"skill::{(s or '').strip()}" for s in skill_names if s]
+        if not skills:
+            continue
+        n = len(skills)
+        n_jd_aligned = 0
+        rsum = 0.0
+        for s in skills:
+            if s in communities and communities[s] in jd_community_ids:
+                n_jd_aligned += 1
+            rsum += idf.get(s, 0.0)
+        purity[i] = n_jd_aligned / n
+        rarity[i] = rsum / n
+        cnode = f"cand::{row['candidate_id']}"
+        deg = G.degree(cnode) if cnode in G else 0
+        degree[i] = float(np.log1p(deg))
+    return purity, rarity, degree
+
+
+if __name__ == "__main__":
+    from ..data import load_or_build_parquet
+    df = load_or_build_parquet().head(1000)
+    coh = career_coherence_scores(df)
+    purity, rarity, deg = skill_community_features(df)
+    print("coherence range:", coh.min(), coh.max())
+    print("purity range:", purity.min(), purity.max())
+    print("rarity range:", rarity.min(), rarity.max())
+    print("degree range:", deg.min(), deg.max())
+
+
+# ---------------------------------------------------------------------------
+# Career evidence score: lexical signal for "shipped ranking/search/recsys"
+# ---------------------------------------------------------------------------
+
+import re as _re
+
+_SHIP_RE = _re.compile(
+    r"\b(built|shipped|launched|migrated|scaled|deployed|productionized|"
+    r"productionised|architected|led|owned|drove|designed|implemented|"
+    r"integrated|rolled out|from zero|to millions|million users|100x|10x)\b",
+    _re.IGNORECASE,
+)
+_RETRIEVAL_RE = _re.compile(
+    r"\b(ranking|ranker|search|retrieval|recommendation|recommender|recsys|"
+    r"ranking system|search system|recommendation system|vector search|"
+    r"hybrid search|elasticsearch|opensearch|faiss|pinecone|weaviate|qdrant|"
+    r"milvus|bm25|embeddings|semantic|learning to rank|learning-to-rank|"
+    r"lambda|lambdarank|ndcg|mrr|map|a/?b|ab test|ab testing|online evaluation|"
+    r"offline evaluation|click.?through|ctr|engagement)\b",
+    _re.IGNORECASE,
+)
+
+
+def career_evidence_score(df: pd.DataFrame) -> np.ndarray:
+    """Lexical score for 'shipped ranking/search/recsys at scale' evidence.
+
+    JD says: "has shipped at least one end-to-end ranking, search, or
+    recommendation system to real users at meaningful scale."
+    We check career descriptions for ship verbs AND retrieval keywords.
+    """
+    out = np.zeros(len(df), dtype=np.float32)
+    df_r = df.reset_index(drop=True)
+
+    def _as_list(v):
+        if v is None:
+            return []
+        if hasattr(v, "tolist"):
+            v = v.tolist()
+        return v if isinstance(v, list) else []
+
+    for i in range(len(df_r)):
+        career = _as_list(df_r.iloc[i].get("career"))
+        if not career:
+            out[i] = 0.0
+            continue
+        # Check last 3 roles
+        recent = career[-3:]
+        n_ship = 0
+        n_ret = 0
+        total = len(recent)
+        for ch in recent:
+            desc = ((ch.get("description") or "") + " " + (ch.get("title") or "")).lower()
+            if _SHIP_RE.search(desc):
+                n_ship += 1
+            if _RETRIEVAL_RE.search(desc):
+                n_ret += 1
+        if total == 0:
+            out[i] = 0.0
+            continue
+        # Score: system match (60%) + ship verb (40%), averaged over roles
+        out[i] = float(min(1.0, 0.6 * (n_ret / total) * 2 + 0.4 * (n_ship / total) * 2))
+    return out

redrob/graph/propagate.py

@@ -0,0 +1,128 @@
+"""Personalised PageRank over the heterogeneous graph.
+
+Supports both the original single-axis PPR and a multi-axis PPR
+(5 JD axes: applied_ml, retrieval_rank, nlp_llm, production_eng, product_company).
+Multi-axis results are cached on disk at artifacts/ppr_axes.pkl.
+"""
+from __future__ import annotations
+
+import pickle
+from pathlib import Path
+from typing import Dict, Iterable, List
+
+import networkx as nx
+
+from .. import config
+from .build import build_graph
+
+
+def _seed_nodes(jd_terms: Iterable[str], prefix: str) -> List[str]:
+    """Map JD terms to graph node labels with the given prefix."""
+    out: List[str] = []
+    for term in jd_terms:
+        key = f"{prefix}::{term.strip()}"
+        out.append(key)
+    return out
+
+
+def personalised_pagerank(
+    jd_terms: Iterable[str] | None = None,
+    alpha: float = 0.85,
+    max_iter: int = 60,
+) -> Dict[str, float]:
+    """Return a dict of {candidate_node: ppr_score}."""
+    G = build_graph()
+    seeds = _seed_nodes(jd_terms or config.CORE_COMPETENCIES, "skill")
+    seeds = [s for s in seeds if s in G]
+    if not seeds:
+        return {}
+    personalization = {n: 0.0 for n in G.nodes()}
+    for s in seeds:
+        personalization[s] = 1.0 / len(seeds)
+    pr = nx.pagerank(
+        G,
+        alpha=alpha,
+        personalization=personalization,
+        max_iter=max_iter,
+        tol=1e-6,
+        weight="weight",
+    )
+    return {n: float(v) for n, v in pr.items()
+            if isinstance(n, str) and n.startswith("cand::")}
+
+
+# Disk cache path for multi-axis PPR
+_PPR_AXES_CACHE = config.ARTIFACTS_DIR / "ppr_axes.pkl"
+
+
+def ppr_axes(
+    axes: Dict[str, List[str]] | None = None,
+    alpha: float = 0.85,
+    max_iter: int = 60,
+) -> Dict[str, Dict[str, float]]:
+    """Multi-axis PPR: one PPR per JD axis.
+
+    Returns Dict[axis_name, Dict[cand_node, ppr_score]].
+    Results are cached at artifacts/ppr_axes.pkl.
+
+    Axes:
+      applied_ml      -> seeds from config.JD_AXES["applied_ml"] on skill:: nodes
+      retrieval_rank  -> seeds from config.JD_AXES["retrieval_rank"] on skill:: nodes
+      nlp_llm         -> seeds from config.JD_AXES["nlp_llm"] on skill:: nodes
+      production_eng   -> seeds from config.JD_AXES["production_eng"] on skill:: nodes
+      product_company  -> seeds from config.JD_AXES["product_company"] on company:: nodes
+    """
+    if axes is None:
+        axes = config.JD_AXES
+
+    # Load cache if present
+    if _PPR_AXES_CACHE.exists():
+        with open(_PPR_AXES_CACHE, "rb") as f:
+            cached: Dict[str, Dict[str, float]] = pickle.load(f)
+        # Validate all axes are present
+        if all(ax in cached for ax in axes):
+            return cached
+
+    G = build_graph()
+    result: Dict[str, Dict[str, float]] = {}
+
+    for axis_name, terms in axes.items():
+        # product_company axis uses company:: nodes; all others use skill::
+        prefix = "company" if axis_name == "product_company" else "skill"
+        seeds = _seed_nodes(terms, prefix)
+        seeds = [s for s in seeds if s in G]
+        if not seeds:
+            result[axis_name] = {}
+            print(f"  ppr_axes[{axis_name}]: 0 seeds found in graph")
+            continue
+        personalization = {n: 0.0 for n in G.nodes()}
+        for s in seeds:
+            personalization[s] = 1.0 / len(seeds)
+        pr = nx.pagerank(
+            G,
+            alpha=alpha,
+            personalization=personalization,
+            max_iter=max_iter,
+            tol=1e-6,
+            weight="weight",
+        )
+        result[axis_name] = {
+            n: float(v)
+            for n, v in pr.items()
+            if isinstance(n, str) and n.startswith("cand::")
+        }
+        top_score = max(result[axis_name].values()) if result[axis_name] else 0.0
+        print(f"  ppr_axes[{axis_name}]: {len(seeds)} seeds, top_score={top_score:.6f}")
+
+    # Persist to disk
+    with open(_PPR_AXES_CACHE, "wb") as f:
+        pickle.dump(result, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+    return result
+
+
+if __name__ == "__main__":
+    axes = ppr_axes()
+    for name, scores in axes.items():
+        top = sorted(scores.items(), key=lambda x: -x[1])[:3]
+        print(f"{name}: {top}")

redrob/rank/__init__.py

@@ -0,0 +1,16 @@
+"""Learning-to-rank with LightGBM LambdaRank and a deterministic fallback."""
+from .train import (
+    build_feature_frame,
+    weak_relevance_label,
+    train_ranker,
+    deterministic_score,
+)
+from .predict import rank_shortlist
+
+__all__ = [
+    "build_feature_frame",
+    "weak_relevance_label",
+    "train_ranker",
+    "deterministic_score",
+    "rank_shortlist",
+]

redrob/rank/predict.py

@@ -0,0 +1,134 @@
+"""Apply the trained ranker to a shortlist and return the final ranking.
+
+Architecture: deterministic-first, with the LTR as a local tiebreaker.
+
+The deterministic composite (see redrob/rank/train.py:deterministic_score)
+was validated empirically to surface 80 elites in top-100, vs 55 for the
+LTR-first approach. The LTR is preserved as a refiner that only reorders
+near-tied deterministic buckets.
+"""
+from __future__ import annotations
+
+from typing import List, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+from .. import config
+from .train import (
+    FEATURE_COLUMNS,
+    build_feature_frame,
+    deterministic_score,
+    train_ranker,
+)
+
+
+# Local-tiebreak threshold (deterministic-score units). Within this band,
+# the LTR refines the ordering; outside it, the deterministic score is the
+# primary key.
+LTR_REFINE_EPS = 0.15
+
+
+def _load_model() -> Tuple[Optional[object], List[str]]:
+    try:
+        return train_ranker(force=False)
+    except Exception:
+        return None, FEATURE_COLUMNS
+
+
+def rank_shortlist(
+    df: pd.DataFrame,
+    use_lgbm: bool = True,
+) -> pd.DataFrame:
+    """Rank a shortlist dataframe.
+
+    Returns the dataframe with `model_score`, `det_score`, `final_score`,
+    and `honeypot_excluded` columns.
+
+    Sort order:
+      1) Honeypot hard-excluded candidates pushed to the bottom.
+      2) Primary sort by deterministic_score (proven composite).
+      3) Within near-tied deterministic buckets (Δ < LTR_REFINE_EPS),
+         re-sort by LTR score. Otherwise keep deterministic order.
+      4) Final tiebreaker: candidate_id ascending.
+    """
+    df = build_feature_frame(df)
+
+    # Always compute deterministic score
+    det = deterministic_score(df)
+    df["det_score"] = det
+    df["model_score"] = det  # for backwards-compat callers
+
+    # Optionally compute LTR score
+    ltr_score: Optional[np.ndarray] = None
+    if use_lgbm:
+        model, feats = _load_model()
+        if model is not None and "sparse_score" in df.columns:
+            # Only run the model if retrieval features are non-zero
+            X = df[feats].fillna(0).astype(np.float32)
+            ltr_score = np.asarray(model.predict(X), dtype=np.float32)
+
+    # Honeypot hard-exclusion flag
+    df["honeypot_excluded"] = df["honeypot_penalty"] >= config.HONEYPOT_HARD_EXCLUDE
+
+    # final_score starts as the deterministic score; honeypot-hard → -1e9
+    df["final_score"] = det.astype(np.float32).copy()
+    df.loc[df["honeypot_excluded"], "final_score"] = -1e9
+
+    # Sort: hard-excluded at bottom, then deterministic-primary, LTR-tiebreak,
+    # candidate_id-asc final tiebreak.
+    df = _sort_with_ltr_refine(df, ltr_score)
+    return df
+
+
+def _sort_with_ltr_refine(
+    df: pd.DataFrame,
+    ltr_score: Optional[np.ndarray],
+) -> pd.DataFrame:
+    """Stable sort with LTR refinement of near-tied deterministic buckets."""
+    n = len(df)
+    if n == 0:
+        return df.reset_index(drop=True)
+
+    # Bucket key: the integer floor of det_score / LTR_REFINE_EPS.
+    # All candidates whose det scores fall in the same bucket are eligible
+    # for LTR-based refinement.
+    eps = float(LTR_REFINE_EPS)
+    if eps <= 0:
+        eps = 0.15
+
+    # Build a stable sort key with two parts:
+    #   primary:   det_score descending
+    #   secondary: (within-bucket rank from LTR) descending
+    #   tertiary:  candidate_id ascending
+    if ltr_score is None or len(ltr_score) != n:
+        # No LTR: just sort by final_score then candidate_id.
+        return df.sort_values(
+            by=["final_score", "candidate_id"],
+            ascending=[False, True],
+            kind="mergesort",
+        ).reset_index(drop=True)
+
+    # Compute the "LTR rank within bucket". For each candidate, find the
+    # set of other candidates within ±eps of its det_score; within that
+    # set, the LTR score determines order.
+    det = df["det_score"].to_numpy()
+    # Sort indices by det descending; ties broken by ltr descending then id asc.
+    # We do this with a stable sort over a composite key.
+    # composite_key = (det_bucket, -ltr, id)  → ascending sort gives the right order
+    # bucket = floor(det / eps). We want highest det first → use -bucket.
+    bucket = np.floor(det / eps).astype(np.int64)
+    order = np.lexsort((
+        df["candidate_id"].to_numpy(),       # tiebreaker 3: candidate_id asc
+        -ltr_score,                          # tiebreaker 2: ltr desc
+        -bucket,                             # primary: bucket desc (i.e., det desc)
+    ))
+    return df.iloc[order].reset_index(drop=True)
+
+
+if __name__ == "__main__":
+    from ..data import load_or_build_parquet
+    df = load_or_build_parquet().head(200)
+    out = rank_shortlist(df)
+    print(out[["candidate_id", "current_title", "title_fit_blend", "det_score",
+               "model_score", "final_score", "honeypot_penalty"]].head(20))

redrob/rank/train.py

@@ -0,0 +1,636 @@
+"""Feature assembly, weak labels, and ranker training.
+
+The ranker is a LightGBM LambdaRank model trained on synthetic weak labels
+derived from the role blueprint. A deterministic scoring function is also
+exposed as a robust fallback that does not depend on the trained model.
+"""
+from __future__ import annotations
+
+import json
+import math
+import re
+from pathlib import Path
+from typing import List, Tuple
+
+import numpy as np
+import pandas as pd
+
+from .. import config
+from ..blueprint import build_blueprint
+from ..data import load_or_build_parquet
+from ..graph.coherence import career_coherence_scores, career_evidence_score, skill_community_features
+from ..graph.propagate import personalised_pagerank, ppr_axes
+from ..features import (
+    title_fit_features,
+    company_tier_features,
+    recruitability_features,
+    honeypot_features,
+    skill_features,
+)
+
+
+# ---------------------------------------------------------------------------
+# Feature assembly
+# ---------------------------------------------------------------------------
+
+def build_feature_frame(df: pd.DataFrame) -> pd.DataFrame:
+    """Compute every feature the ranker needs for the given dataframe."""
+    bp = build_blueprint()
+
+    # 1) Graph PPR over the full 100k pool (cached inside the graph module)
+    ppr = personalised_pagerank()
+    df = df.copy()
+    df["ppr_score"] = df["candidate_id"].map(lambda c: ppr.get(f"cand::{c}", 0.0))
+
+    # 1b) Multi-axis PPR: 5 specialised axes for better discrimination
+    ppr_ax = ppr_axes()
+    for axis_name, axis_scores in ppr_ax.items():
+        df[f"ppr_{axis_name}"] = df["candidate_id"].map(
+            lambda c, ax=axis_scores: ax.get(f"cand::{c}", 0.0)
+        )
+    # Derived: best axis score, mean, and std (breadth of fit)
+    ppr_axis_cols = [f"ppr_{a}" for a in ppr_ax.keys()]
+    if ppr_axis_cols:
+        ppr_vals = df[ppr_axis_cols].astype(float)
+        df["ppr_max_axis"] = ppr_vals.max(axis=1)
+        df["ppr_axis_mean"] = ppr_vals.mean(axis=1)
+        df["ppr_axis_std"] = ppr_vals.std(axis=1)
+
+    # Log-scaled PPR features (raw values are ~1e-6, invisible to the LTR)
+    df["ppr_score_log"] = np.log1p(df["ppr_score"].astype(float).to_numpy() * 1e5)
+    for col in ppr_axis_cols + ["ppr_max_axis", "ppr_axis_mean", "ppr_axis_std"]:
+        if col in df.columns:
+            df[f"{col}_log"] = np.log1p(df[col].astype(float).to_numpy() * 1e5)
+
+    # 2) Skill community features
+    purity, rarity, degree = skill_community_features(df)
+    df["skill_community_purity"] = purity
+    df["skill_rarity"] = rarity
+    df["graph_degree"] = degree
+
+    # 3) Career coherence
+    df["career_coherence"] = career_coherence_scores(df)
+
+    # 3b) Career evidence: shipped ranking/search/recsys signal
+    df["career_evidence"] = career_evidence_score(df)
+
+    # 4) Title features
+    tfeat = title_fit_features(df).reset_index(drop=True)
+    df = pd.concat([df.reset_index(drop=True), tfeat], axis=1)
+
+    # 4b) Company tier
+    ctier = company_tier_features(df).reset_index(drop=True)
+    df = pd.concat([df.reset_index(drop=True), ctier], axis=1)
+
+    # 5) Behavioral
+    bfeat = recruitability_features(df).reset_index(drop=True)
+    df = pd.concat([df, bfeat], axis=1)
+
+    # 6) Honeypot
+    hfeat = honeypot_features(df).reset_index(drop=True)
+    df = pd.concat([df, hfeat], axis=1)
+
+    # 7) Skill features
+    sfeat = skill_features(df).reset_index(drop=True)
+    df = pd.concat([df, sfeat], axis=1)
+
+    # 7b) Negative-specification flags (JD "do NOT want") — after skill features for n_core_skills
+    yoe_arr = df["yoe"].astype(float).to_numpy()
+
+    def _as_list_local(v):
+        if v is None:
+            return []
+        if hasattr(v, "tolist"):
+            return v.tolist()
+        return v if isinstance(v, list) else []
+
+    # Compute avg tenure and n_titles directly
+    avg_ten_arr = np.zeros(len(df), dtype=float)
+    n_tit_arr = np.zeros(len(df), dtype=np.int32)
+    is_consulting_only_arr = np.zeros(len(df), dtype=np.float32)
+    for i in range(len(df)):
+        ch = _as_list_local(df.iloc[i].get("career"))
+        durs = [int(c.get("duration_months") or 0) for c in ch]
+        durs = [d for d in durs if d > 0]
+        avg_ten_arr[i] = sum(durs) / len(durs) if durs else 0.0
+        ht = _as_list_local(df.iloc[i].get("career_titles"))
+        n_tit_arr[i] = len(ht)
+        companies = _as_list_local(df.iloc[i].get("career_companies"))
+        is_consulting_only_arr[i] = float(
+            all((c or "").strip() in config.IT_SERVICES_PURE_PLAY for c in companies)
+            if companies else False
+        )
+
+    is_title_chaser = ((avg_ten_arr < 18.0) & (n_tit_arr >= 4) & (yoe_arr >= 5.0)).astype(np.float32)
+
+    # Framework enthusiast (refined by super-smart-agent review).
+    # Old rule: framework_skills >= 2 AND core_skills < 3.
+    # New rule: framework_skills >= 2 AND core_skills < 3 AND
+    #          (retrieval/eval fundamentals weak: BM25/NDCG/LtR/FAISS
+    #           count < 2) AND (production evidence < 0.3).
+    # This catches candidates with high LangChain/LlamaIndex use but
+    # without evidence of building the underlying retrieval/eval systems.
+    n_core_arr_val = df["n_core_skills"].astype(float).to_numpy()
+    _FW_SET = {"langchain", "llamaindex", "langgraph"}
+    _RETRIEVAL_EVAL_KW = {
+        "bm25", "faiss", "pinecone", "weaviate", "qdrant", "milvus",
+        "elasticsearch", "opensearch", "ndcg", "mrr", "map",
+        "learning to rank", "lambdarank", "sentence transformers",
+        "bge", "e5",
+    }
+    fw_score = np.zeros(len(df), dtype=np.float32)
+    ret_eval_count = np.zeros(len(df), dtype=np.int32)
+    for i in range(len(df)):
+        skill_names = _as_list_local(df.iloc[i].get("skill_names"))
+        names_lc = {(s or "").lower() for s in skill_names}
+        fw_score[i] = sum(1 for nm in names_lc if nm in _FW_SET)
+        ret_eval_count[i] = sum(1 for nm in names_lc if nm in _RETRIEVAL_EVAL_KW)
+
+    # Production evidence
+    car_ev_arr = np.zeros(len(df), dtype=np.float32)
+    if "career_evidence" in df.columns:
+        car_ev_arr = df["career_evidence"].fillna(0.0).astype(float).to_numpy()
+
+    is_framework_enthusiast = (
+        (fw_score >= 2)
+        & (n_core_arr_val < 3)
+        & (ret_eval_count < 2)
+        & (car_ev_arr < 0.3)
+    ).astype(np.float32)
+
+    df["is_title_chaser"] = is_title_chaser
+    df["is_consulting_only"] = is_consulting_only_arr
+    df["is_framework_enthusiast"] = is_framework_enthusiast
+
+    # 8) Seniority / location / industry features
+    df["years_in_ideal_band"] = (
+        (df["yoe"] >= config.YOE_IDEAL_LOW) & (df["yoe"] <= config.YOE_IDEAL_HIGH)
+    ).astype(np.float32)
+    df["yoe_log"] = np.log1p(df["yoe"].clip(lower=0))
+
+    # Seniority: senior/staff/principal/lead vs junior
+    _cur_titles = df["current_title"].fillna("").str.lower().tolist()
+    _senior_re = re.compile(r"\b(senior|staff|principal|lead|head|chief|architect)\b", re.IGNORECASE)
+    _junior_re = re.compile(r"\b(junior|jr\.?|intern|trainee|associate)\b", re.IGNORECASE)
+    df["is_senior_title"] = np.array(
+        [1.0 if _senior_re.search(t) else 0.0 for t in _cur_titles], dtype=np.float32
+    )
+    df["is_junior_title"] = np.array(
+        [1.0 if _junior_re.search(t) else 0.0 for t in _cur_titles], dtype=np.float32
+    )
+    df["preferred_location"] = df["location"].fillna("").apply(
+        lambda s: int(any(loc.lower() in (s or "").lower() for loc in config.PREFERRED_LOCATIONS))
+    )
+    df["country_ok"] = df["country"].fillna("").apply(
+        lambda s: int(s.strip() in config.COUNTRY_OK)
+    )
+
+    def _as_dict(s):
+        if s is None:
+            return {}
+        if hasattr(s, "tolist"):
+            s = s.tolist() if s else {}
+        return s if isinstance(s, dict) else {}
+
+    df["willing_to_relocate"] = df["signals"].apply(
+        lambda s: int(bool(_as_dict(s).get("willing_to_relocate", False)))
+    )
+    df["notice_period_days"] = df["signals"].apply(
+        lambda s: int(_as_dict(s).get("notice_period_days", 90))
+    )
+    df["notice_log"] = np.log1p(df["notice_period_days"].clip(lower=0))
+    df["work_mode_remote"] = df["signals"].apply(
+        lambda s: int(_as_dict(s).get("preferred_work_mode", "") in ("remote", "flexible", "hybrid"))
+    )
+
+    # 9) Industry / company features
+    def _as_list(v):
+        if v is None:
+            return []
+        if hasattr(v, "tolist"):
+            return v.tolist()
+        return v
+
+    df["is_services_company"] = df["career_companies"].apply(
+        lambda lst: int(any((c or "").strip() in config.IT_SERVICES_PURE_PLAY for c in (_as_list(lst) or [])))
+    )
+    df["is_product_company"] = df["career_companies"].apply(
+        lambda lst: int(any((c or "").strip() in config.PRODUCT_COMPANY_HINTS for c in (_as_list(lst) or [])))
+    )
+    df["n_industries"] = df["career_industries"].apply(
+        lambda lst: len(set(_as_list(lst) or []))
+    )
+
+    return df
+
+
+FEATURE_COLUMNS: List[str] = [
+    # retrieval
+    "sparse_score", "dense_score", "rrf_score", "structured_hit",
+    # graph — raw and log-scaled (the raw PPR values are ~1e-6 noise scale;
+    # the LTR benefits enormously from seeing human-scale features)
+    "ppr_score",
+    "ppr_applied_ml", "ppr_retrieval_rank", "ppr_nlp_llm",
+    "ppr_production_eng", "ppr_product_company",
+    "ppr_max_axis", "ppr_axis_mean", "ppr_axis_std",
+    "ppr_score_log",
+    "ppr_applied_ml_log", "ppr_retrieval_rank_log", "ppr_nlp_llm_log",
+    "ppr_production_eng_log", "ppr_product_company_log",
+    "ppr_max_axis_log", "ppr_axis_mean_log", "ppr_axis_std_log",
+    "skill_community_purity", "skill_rarity", "graph_degree",
+    "career_coherence", "career_evidence",
+    # title
+    "title_weight", "title_history_max_weight", "is_target_title",
+    "is_noneng_title", "is_data_platform", "is_data_science", "is_generic_swe",
+    "n_career_titles", "avg_tenure_months", "title_fit_blend",
+    # skills
+    "skill_count", "n_core_skills", "n_adj_skills", "n_neg_skills",
+    "n_advanced_skills", "n_expert_skills", "must_have_coverage",
+    "adjacent_coverage", "ai_signal_strength", "assessment_max",
+    "assessment_mean", "endorsement_log_mean", "duration_log_mean",
+    "jaccard_core", "jd_criticality_score",
+    # behavioral
+    "recruit_open_to_work", "recruit_response_rate", "recruit_verified",
+    "recruit_completeness", "recruit_recency", "recruit_notice_ok",
+    "recruit_recruiter_saves", "recruit_interview_completion",
+    "recruit_offer_acceptance", "recruit_github", "recruitability",
+    # honeypot
+    "honeypot_penalty",
+    # seniority / location / industry
+    "yoe", "years_in_ideal_band", "yoe_log", "preferred_location",
+    "country_ok", "willing_to_relocate", "notice_period_days", "notice_log",
+    "work_mode_remote", "is_services_company", "is_product_company",
+    "n_industries", "company_tier_current", "company_tier_max",
+    "is_top_tier_company", "is_product_company_v2",
+    "is_title_chaser", "is_consulting_only", "is_framework_enthusiast",
+    # seniority
+    "is_senior_title", "is_junior_title",
+]
+
+
+# ---------------------------------------------------------------------------
+# Weak labels (used to train the LambdaRank model)
+# ---------------------------------------------------------------------------
+
+def weak_relevance_label(df: pd.DataFrame) -> np.ndarray:
+    """A 0..4 relevance grade derived from a transparent composite.
+
+    The grade is intentionally not the same as `deterministic_score` — it
+    is a *learning target* that emphasises the role blueprint's
+    hard requirements (target title, must-have coverage, low honeypot
+    penalty) so the ranker can calibrate.
+
+    Enriched with: multi-axis PPR diversity (breadth), career evidence,
+    company tier. Thresholds biased to be more discriminative at the
+    top end (tier 4) so the LTR learns to distinguish gold candidates.
+    """
+    title_w = df["title_fit_blend"].astype(float).to_numpy()
+    must = df["must_have_coverage"].astype(float).to_numpy()
+    ppr = df["ppr_score"].astype(float).to_numpy()
+    ppr_max = df["ppr_max_axis"].astype(float).to_numpy() if "ppr_max_axis" in df.columns else ppr
+    # PPR breadth: standard deviation across the 5 axes
+    ppr_axis_cols = [c for c in df.columns
+                     if c.startswith("ppr_") and c not in ("ppr_score", "ppr_max_axis",
+                                                              "ppr_axis_mean", "ppr_axis_std")]
+    if ppr_axis_cols:
+        ppr_breadth = df[ppr_axis_cols].astype(float).std(axis=1).to_numpy()
+    else:
+        ppr_breadth = np.zeros(len(df))
+    coh = df["career_coherence"].astype(float).to_numpy()
+    car_ev = df["career_evidence"].astype(float).to_numpy()
+    rec = df["recruitability"].astype(float).to_numpy()
+    hp = df["honeypot_penalty"].astype(float).to_numpy()
+    jac = df["jaccard_core"].astype(float).to_numpy()
+    yoe = df["yoe"].astype(float).to_numpy()
+    in_band = df["years_in_ideal_band"].astype(float).to_numpy()
+
+    # Company tier (only if present)
+    if "company_tier_max" in df.columns:
+        ctier = df["company_tier_max"].astype(float).to_numpy()
+    else:
+        ctier = np.zeros(len(df))
+
+    score = (
+        5.0 * title_w
+        + 2.0 * must
+        + 1.5 * np.log1p(ppr_max * 1e5)  # multi-axis best axis, log-scaled
+        + 0.8 * ppr_breadth * 1e5         # breadth of fit (axes std)
+        + 1.0 * coh
+        + 1.0 * rec
+        - 3.0 * hp
+        + 0.5 * jac
+        + 0.5 * np.log1p(yoe) * in_band
+        + 0.5 * ctier / 3.0               # company tier contribution
+        + 0.3 * df["is_product_company"].astype(float).to_numpy()
+        + 1.5 * df["jd_criticality_score"].astype(float).to_numpy()
+        + 0.7 * car_ev
+        # Negative-spec penalties (soft)
+        - 0.8 * (df["is_title_chaser"].astype(float).to_numpy()
+                  if "is_title_chaser" in df.columns
+                  else np.zeros(len(df)))
+        - 0.5 * (df["is_consulting_only"].astype(float).to_numpy()
+                   if "is_consulting_only" in df.columns
+                   else np.zeros(len(df)))
+    )
+    # Convert to 5-point Likert (tuned for better discrimination at top)
+    grade = np.zeros(len(df), dtype=np.int32)
+    grade[score > 2.0] = 1
+    grade[score > 3.5] = 2
+    grade[score > 5.0] = 3
+    grade[score > 6.5] = 4
+    return grade
+
+
+# ---------------------------------------------------------------------------
+# Deterministic fallback scorer (works without training)
+# ---------------------------------------------------------------------------
+
+def deterministic_score(df: pd.DataFrame) -> np.ndarray:
+    """JD-tuned deterministic composite.
+
+    Validated empirically: produces 80 elites in top-100 (vs 55 for the
+    v3 LTR-first ranking). Upweights behavior (JD: "active on Redrob
+    so we can talk to them"), adds seniority term (Senior/Staff/Lead
+    vs Junior), keeps honeypot multiplicative exclusion (the >10% DQ
+    filter), keeps JD-criticality, multi-axis PPR, career evidence.
+    """
+    s = np.zeros(len(df), dtype=np.float32)
+
+    # Title (heavy weight — JD emphasises this)
+    s += 4.5 * df["title_fit_blend"].astype(float).to_numpy()
+    s += 1.5 * df["is_target_title"].astype(float).to_numpy()
+    s -= 2.0 * df["is_noneng_title"].astype(float).to_numpy()
+
+    # Seniority term: senior/staff/principal/lead up, junior down
+    if "is_senior_title" in df.columns:
+        s += 0.5 * df["is_senior_title"].astype(float).to_numpy()
+        s -= 1.0 * df["is_junior_title"].astype(float).to_numpy()
+
+    # Skills
+    s += 2.0 * df["must_have_coverage"].astype(float).to_numpy()
+    s += 0.5 * df["adjacent_coverage"].astype(float).to_numpy()
+    s += 0.3 * df["jaccard_core"].astype(float).to_numpy()
+    s += 0.4 * (df["assessment_max"].fillna(0).astype(float).to_numpy() / 100.0)
+    s += 0.4 * (df["assessment_mean"].fillna(0).astype(float).to_numpy() / 100.0)
+    # JD-criticality: RAG/FAISS/BM25/evaluation 2× weight
+    s += 1.5 * df["jd_criticality_score"].fillna(0).astype(float).to_numpy()
+
+    # Graph
+    s += 1.5 * np.log1p(df["ppr_score"].astype(float).to_numpy() * 1e5)
+    # Multi-axis PPR: best axis score + breadth bonus
+    if "ppr_max_axis" in df.columns:
+        max_ax = df["ppr_max_axis"].astype(float).to_numpy()
+        std_ax = df["ppr_axis_std"].astype(float).to_numpy()
+        std_max = std_ax.max() + 1e-9
+        s += 0.6 * np.log1p(max_ax * 1e5)
+        s += 0.3 * (std_ax / std_max)  # breadth of fit bonus
+    s += 0.8 * df["career_coherence"].astype(float).to_numpy()
+    s += 0.8 * df["career_evidence"].astype(float).to_numpy()
+    s += 0.5 * df["skill_community_purity"].astype(float).to_numpy()
+    s += 0.2 * df["skill_rarity"].astype(float).to_numpy()
+
+    # Behavior — JD: "active on Redrob so we can actually talk to them"
+    s += 2.0 * df["recruitability"].astype(float).to_numpy()
+    # Per-feature bonuses for the most important JD signals
+    if "recruit_recruiter_saves" in df.columns:
+        s += 0.5 * df["recruit_recruiter_saves"].astype(float).to_numpy()
+    if "recruit_response_rate" in df.columns:
+        s += 0.4 * df["recruit_response_rate"].fillna(0).astype(float).to_numpy()
+
+    # Honeypot penalty — multiplicative exclusion (the >10% DQ filter)
+    hp = df["honeypot_penalty"].astype(float).to_numpy()
+    s -= 5.0 * hp
+    # Hard push to -inf for honeypot-hard
+    s[hp >= config.HONEYPOT_HARD_EXCLUDE] = -1e9
+
+    # JD "do NOT want" soft penalties
+    if "is_title_chaser" in df.columns:
+        s -= 0.6 * df["is_title_chaser"].astype(float).to_numpy()
+        s -= 0.4 * df["is_framework_enthusiast"].astype(float).to_numpy()
+        s -= 0.5 * df["is_consulting_only"].astype(float).to_numpy()
+
+    # Seniority & location
+    s += 0.4 * df["years_in_ideal_band"].astype(float).to_numpy()
+    s += 0.3 * df["country_ok"].astype(float).to_numpy()
+    s += 0.2 * df["preferred_location"].astype(float).to_numpy()
+    s += 0.2 * df["willing_to_relocate"].astype(float).to_numpy()
+    s -= 0.4 * (df["notice_period_days"].fillna(90).clip(lower=0).astype(float).to_numpy() / 180.0)
+    s += 0.3 * df["is_product_company"].astype(float).to_numpy()
+    s -= 0.2 * df["is_services_company"].astype(float).to_numpy() * (1.0 - df["must_have_coverage"].astype(float).to_numpy())
+    # Company tier: tier-3 company bonus
+    if "company_tier_max" in df.columns:
+        s += 0.4 * df["company_tier_max"].astype(float).to_numpy() / 3.0
+        s += 0.2 * df["is_top_tier_company"].astype(float).to_numpy()
+
+    # Retrieval signals (additive)
+    if "rrf_score" in df.columns:
+        s += 0.6 * df["rrf_score"].fillna(0).astype(float).to_numpy()
+    if "sparse_score" in df.columns:
+        s += 0.05 * (df["sparse_score"].fillna(0).astype(float).to_numpy() / 30.0)
+    if "dense_score" in df.columns:
+        s += 0.6 * df["dense_score"].fillna(0).astype(float).to_numpy()
+
+    # ----------------------------------------------------------------------
+    # Conservative availability penalty (added by super-smart-agent review).
+    # JD explicitly says "perfect-on-paper but inactive" candidates should
+    # be down-weighted. Apply a soft penalty only when MULTIPLE risk signals
+    # fire simultaneously — never on a single signal (too noisy).
+    # Trigger conditions: low response rate AND stale activity AND long notice.
+    # Penalty magnitudes kept small so top-10 is unaffected.
+    # ----------------------------------------------------------------------
+    # v10: Added Tier 2 (low response + low recruitability) to catch the
+    # "strong profile but actively unreachable" archetype that Tier 1 misses
+    # when recency/notice happen to be OK on paper.
+    try:
+        rr = df["recruit_response_rate"].fillna(0.0).astype(float).to_numpy()
+        notice = df["notice_period_days"].fillna(90).astype(float).to_numpy()
+        recency = df["recruit_recency"].fillna(0.0).astype(float).to_numpy()
+
+        # Risk flags (0/1)
+        low_rr = (rr < 0.15).astype(np.float32)         # < 15% response rate
+        stale_activity = (recency < 0.20).astype(np.float32)  # recency<0.2 means old
+        long_notice = (notice > 90).astype(np.float32)         # > 90 days notice
+
+        # Conservative: only penalise when ≥2 risk signals fire
+        n_risks = low_rr + stale_activity + long_notice
+        # 0 risks → 0, 1 risk → 0, 2 risks → 0.5, 3 risks → 1.0
+        risk_severity = np.maximum(n_risks - 1, 0) / 2.0
+        # Penalty: at most -1.5 (3 risks); only top-50 affected in practice
+        s -= 1.5 * risk_severity
+
+        # v10 Tier 2: low response + low recruitability composite. This is a
+        # SINGLE-signal rule but is the most informative single signal — the
+        # recruitability composite already aggregates response+notice+recency,
+        # so low recruitability + low response together is a clear "unreachable"
+        # signal. Magnitude 0.4 keeps it additive (won't dethrone top-10, which
+        # all have response > 0.15 and recruitability > 0.50).
+        if "recruitability" in df.columns:
+            rec = df["recruitability"].fillna(0.5).astype(float).to_numpy()
+            bad_contact = ((rr < 0.15) & (rec < 0.50)).astype(np.float32)
+            s -= 0.4 * bad_contact
+    except Exception:
+        # Be defensive — availability penalty is a soft signal, must not break ranking
+        pass
+
+    # ----------------------------------------------------------------------
+    # v10: Non-India + not-willing-to-relocate penalty.
+    # JD says India preferred, outside India is case-by-case, no visa
+    # sponsorship. Not a hard exclusion (case-by-case), but a moderate penalty
+    # so they don't outrank India-based candidates with comparable profiles.
+    # Magnitude 0.5 ≈ 2-3 ranks of typical gap; leaves top-10 unchanged
+    # (all top-10 are India-based).
+    # ----------------------------------------------------------------------
+    try:
+        country_arr = df["country"].fillna("").astype(str).to_numpy()
+        reloc = df["willing_to_relocate"].fillna(0).astype(float).to_numpy()
+        non_india_no_reloc = ((country_arr != "India") & (reloc == 0)).astype(np.float32)
+        s -= 0.5 * non_india_no_reloc
+    except Exception:
+        pass
+
+    return s
+
+
+# ---------------------------------------------------------------------------
+# Train LightGBM ranker
+# ---------------------------------------------------------------------------
+
+
+def _populate_real_retrieval_scores(df: pd.DataFrame) -> pd.DataFrame:
+    """Populate sparse_score / rrf_score / structured_hit / dense_score
+    with real (non-zero) values from BM25 + RRF + structured-gate over
+    the entire 100k pool.
+
+    Pre-compute — allowed to exceed the 5min Stage 3 budget because only
+    the ranking step in scripts/run_ranking.py must fit the budget. The
+    LTR's value is bounded by what features it sees, and 90% importance
+    was previously on noise-scale PPR columns because retrieval features
+    were zero-filled. This fixes that.
+    """
+    import time as _time
+
+    def _log(msg: str) -> None:
+        print(f"  [retrieval-prep] {msg}", flush=True)
+
+    t0 = _time.time()
+    df = df.reset_index(drop=True)
+    n = len(df)
+
+    # BM25 over the pool
+    from ..retrieval.bm25 import build_bm25, bm25_search
+    bm25_obj = build_bm25(force=False)
+    sparse_hits = bm25_search(
+        config.JD_QUERY_TERMS, top_k=config.BM25_TOP_K, bm25_obj=bm25_obj
+    )
+    _log(f"BM25 done ({_time.time()-t0:.1f}s)")
+
+    # Structured-gate as soft score
+    gate_terms_lc = {t.lower() for t in config.STRUCTURED_GATE_TERMS}
+
+    def _gate_text_score(text):
+        if not text:
+            return 0.0
+        tl = text.lower()
+        n_hits = sum(1 for g in gate_terms_lc if g in tl)
+        return float(min(1.0, n_hits / 5.0))
+
+    def _gate_skill_score(skills):
+        if skills is None:
+            return 0.0
+        if hasattr(skills, "tolist"):
+            skills = skills.tolist()
+        n = sum(1 for s in (skills or []) if (s or "").lower() in gate_terms_lc)
+        return float(min(1.0, n / 3.0))
+
+    text_scores = df["text_corpus"].fillna("").apply(_gate_text_score).to_numpy()
+    skill_scores = df["skill_names"].apply(_gate_skill_score).to_numpy()
+    gate_score = np.maximum(text_scores, skill_scores).astype(np.float32)
+
+    # Aggregate BM25 max score per candidate (over all queries)
+    sparse_max: dict = {}
+    for ranking in sparse_hits.values():
+        for cid, sc in ranking:
+            if sc > sparse_max.get(cid, 0.0):
+                sparse_max[cid] = sc
+
+    # RRF: same logic as run_ranking.py
+    from ..retrieval.rrf import rrf_fuse
+    rankings = list(sparse_hits.values())
+    fused = rrf_fuse(
+        rankings, k=config.RRF_K, top_n=n,
+        channel_weights=[1.0] * len(rankings),
+    )
+    rrf_max: dict = {cid: float(s) for cid, s, _ in fused}
+
+    df["sparse_score"] = df["candidate_id"].map(lambda c: float(sparse_max.get(c, 0.0))).astype(np.float32)
+    df["rrf_score"] = df["candidate_id"].map(lambda c: float(rrf_max.get(c, 0.0))).astype(np.float32)
+    df["structured_hit"] = gate_score  # already a numpy float32 array aligned to df
+    # Dense not pre-computed here (would dominate runtime); leave 0 → the LTR
+    # will learn that this feature is uninformative when absent at training.
+    if "dense_score" not in df.columns:
+        df["dense_score"] = 0.0
+    _log(f"all retrieval cols populated ({_time.time()-t0:.1f}s)")
+    return df
+
+
+# ---------------------------------------------------------------------------
+
+def train_ranker(force: bool = False) -> Tuple[object, List[str]]:
+    """Train a LambdaRank model on the full 100k pool using weak labels.
+
+    Returns (model, feature_columns).
+    """
+    if not force and config.RANKER_TXT.exists():
+        import lightgbm as lgb
+        return lgb.Booster(model_file=str(config.RANKER_TXT)), FEATURE_COLUMNS
+
+    df = load_or_build_parquet()
+    df = build_feature_frame(df)
+    # Pre-compute real retrieval scores for the entire 100k pool so the LTR
+    # sees non-zero signals for sparse_score / rrf_score / structured_hit /
+    # dense_score. This is pre-compute (allowed to exceed 5min); only the
+    # ranking step that produces the CSV must fit the budget.
+    df = _populate_real_retrieval_scores(df)
+    df["label"] = weak_relevance_label(df)
+
+    # LambdaRank requires per-group rows <= 10000. We split the 100k pool
+    # into ~20 pseudo-queries (groups of 5000 random candidates) so the
+    # ranker can learn a global ordering while respecting the limit.
+    n = len(df)
+    rng = np.random.default_rng(42)
+    perm = rng.permutation(n)
+    group_size = 5000
+    n_groups = (n + group_size - 1) // group_size
+    groups: list = []
+    for g in range(n_groups):
+        start = g * group_size
+        end = min(n, start + group_size)
+        groups.append(end - start)
+    # Reorder dataframe by the permutation so groups are contiguous
+    df = df.iloc[perm].reset_index(drop=True)
+    y = df["label"].astype(np.int32).to_numpy()
+    X = df[FEATURE_COLUMNS].fillna(0).astype(np.float32)
+
+    import lightgbm as lgb  # type: ignore
+
+    train_data = lgb.Dataset(X, label=y, group=groups)
+    model = lgb.train(
+        config.RANKER_PARAMS,
+        train_data,
+        valid_sets=[train_data],
+        valid_names=["train"],
+    )
+    model.save_model(str(config.RANKER_TXT))
+    return model, FEATURE_COLUMNS
+
+
+if __name__ == "__main__":
+    import sys, time
+    t = time.time()
+    model, feats = train_ranker(force=True)
+    print(f"trained in {time.time()-t:.1f}s, features={len(feats)}")
+    df = load_or_build_parquet().head(5)
+    df = build_feature_frame(df)
+    X = df[feats].fillna(0).astype(np.float32)
+    print("scores:", model.predict(X))
+    print("det:", deterministic_score(df))

redrob/reasoning/__init__.py

@@ -0,0 +1,9 @@
+"""Deterministic, evidence-driven reasoning strings for each top-100 candidate.
+
+No LLM is used at submission time. The reasoning text is composed from a
+small library of templates that cite specific facts from the candidate's
+profile and connect them to JD requirements.
+"""
+from .template import generate_reasoning
+
+__all__ = ["generate_reasoning"]

redrob/reasoning/template.py

@@ -0,0 +1,598 @@
+"""Deterministic reasoning template generator — rich, JD-specific, evidence-grounded.
+
+Each reasoning is composed of 3-4 short clauses that each answer a SPECIFIC
+question the Stage 4 manual reviewer will ask:
+  1) Who is this candidate and what have they shipped? (career evidence)
+  2) Which JD requirements does their profile satisfy? (skill→JD mapping)
+  3) What's the fit quality — gold/silver/bronze/long-shot?
+  4) What concerns should the reviewer verify?
+
+The template library uses rank-band × company-tier × evidence-quality cells,
+each containing 3-5 distinct clause templates so the Stage 4 Variation check
+sees substantial diversity (not the same "Senior AI Engineer fit:" 100 times).
+"""
+from __future__ import annotations
+
+import re
+from typing import Dict, List, Tuple
+
+import pandas as pd
+
+from .. import config
+
+
+# ---------------------------------------------------------------------------
+# Company context — one-line industry/product description
+# ---------------------------------------------------------------------------
+# Helps the reviewer understand WHY a candidate at a given company is a fit.
+
+_COMPANY_CONTEXT: Dict[str, str] = {
+    # Tier-3 global tech
+    "Google": "global search/AI platform",
+    "Meta": "global social-AI platform",
+    "Microsoft": "global enterprise/cloud+AI platform",
+    "Amazon": "global e-commerce/cloud platform",
+    "Apple": "global consumer hardware/ML platform",
+    "Netflix": "global streaming with ML-driven recommendations",
+    "Stripe": "global fintech infrastructure",
+    "Airbnb": "global marketplace with search/ranking",
+    "Uber": "global marketplace with real-time ML",
+    "LinkedIn": "professional network with search/ranking",
+    "Salesforce": "enterprise SaaS with AI features",
+    "Adobe": "creative tools with generative AI",
+    "NVIDIA": "GPU/ML infrastructure",
+    "Snowflake": "data cloud platform",
+    "Databricks": "data/ML platform",
+    "OpenAI": "frontier AI lab",
+    "Anthropic": "frontier AI lab",
+    "Cohere": "enterprise LLM platform",
+    "Hugging Face": "open-source ML platform",
+    "Pinecone": "vector database company",
+    "Weaviate": "open-source vector database",
+    "Qdrant": "open-source vector database",
+    # Tier-2 Indian product
+    "Razorpay": "Indian fintech (payments)",
+    "Zerodha": "Indian fintech (brokerage)",
+    "CRED": "Indian fintech (credit card rewards)",
+    "PhonePe": "Indian fintech (UPI payments)",
+    "Paytm": "Indian fintech (payments+banking)",
+    "Swiggy": "Indian food delivery marketplace",
+    "Zomato": "Indian food delivery marketplace",
+    "Flipkart": "Indian e-commerce",
+    "Meesho": "Indian social commerce",
+    "Urban Company": "Indian services marketplace",
+    "Dream11": "Indian fantasy sports platform",
+    "Groww": "Indian investing platform",
+    "Cars24": "Indian used-car marketplace",
+    "Postman": "API development platform",
+    "Freshworks": "SaaS/CRM platform",
+    "Zoho": "Indian SaaS suite",
+    "Chargebee": "Indian subscription billing SaaS",
+    "BrowserStack": "Indian testing infrastructure SaaS",
+    "Sprinklr": "customer experience platform",
+    "CleverTap": "customer engagement platform",
+    "MoEngage": "customer engagement platform",
+    "ShareChat": "Indian social media",
+    "InMobi": "Indian ad-tech",
+    "Ola": "Indian ride-hailing",
+    "Rivigo": "Indian logistics",
+    "Delhivery": "Indian logistics",
+    "Udaan": "Indian B2B commerce",
+    "Oyo": "Indian hospitality",
+    # Indian AI-first labs
+    "Sarvam AI": "Indian AI-first lab (foundation models)",
+    "Niramai": "Indian health-tech AI",
+    "Rephrase": "Indian generative video AI",
+    "Fractal": "Indian AI/analytics services",
+    "Genpact": "Indian AI services + product",
+    "Wipro": "IT services",
+    "TCS": "IT services",
+    "Infosys": "IT services",
+}
+
+
+def _company_context(company: str) -> str:
+    """One-line context for a known company, else empty."""
+    c = (company or "").strip()
+    if not c:
+        return ""
+    for name, ctx in _COMPANY_CONTEXT.items():
+        if name.lower() in c.lower() or c.lower() in name.lower():
+            return ctx
+    return ""
+
+
+# ---------------------------------------------------------------------------
+# Skill → JD-requirement mapping
+# ---------------------------------------------------------------------------
+
+# Each entry: skill (lowercase) → (why_it_matters, jd_section)
+_JD_SKILL_MEANING: Dict[str, Tuple[str, str]] = {
+    "rag": ("RAG is the JD's central retrieval-augmented generation use case", "absolutely need"),
+    "retrieval augmented generation": ("RAG is central to the JD's product", "absolutely need"),
+    "faiss": ("FAISS is one of the JD's required vector databases", "absolutely need"),
+    "pinecone": ("Pinecone is one of the JD's required vector databases", "absolutely need"),
+    "weaviate": ("Weaviate is one of the JD's required vector databases", "absolutely need"),
+    "qdrant": ("Qdrant is one of the JD's required vector databases", "absolutely need"),
+    "milvus": ("Milvus is one of the JD's required vector databases", "absolutely need"),
+    "opensearch": ("OpenSearch is one of the JD's required retrieval engines", "absolutely need"),
+    "elasticsearch": ("Elasticsearch is one of the JD's required retrieval engines", "absolutely need"),
+    "bm25": ("BM25 is the JD's required sparse-retrieval primitive", "absolutely need"),
+    "hybrid search": ("Hybrid search is the JD's central retrieval pattern", "absolutely need"),
+    "dense retrieval": ("Dense retrieval is central to the JD's hybrid search", "absolutely need"),
+    "sentence transformers": ("sentence-transformers experience is core to the JD", "absolutely need"),
+    "bge": ("BGE encoder experience aligns with the JD's embedding layer", "absolutely need"),
+    "e5": ("E5 encoder experience aligns with the JD's embedding layer", "absolutely need"),
+    "embeddings": ("Embedding experience is core to the JD's retrieval", "absolutely need"),
+    "learning to rank": ("Learning-to-rank is a JD's required evaluation capability", "absolutely need"),
+    "lambdarank": ("LambdaRank directly applies to the JD's ranking pipeline", "absolutely need"),
+    "ndcg": ("NDCG is the JD's required ranking evaluation metric", "absolutely need"),
+    "mrr": ("MRR is the JD's required evaluation metric", "absolutely need"),
+    "map": ("MAP is the JD's required evaluation metric", "absolutely need"),
+    "pytorch": ("PyTorch is the JD's expected deep learning framework", "absolutely need"),
+    "tensorflow": ("TensorFlow experience is bonus for the JD", "nice to have"),
+    "transformers": ("HuggingFace Transformers experience is core to the JD", "absolutely need"),
+    "llm": ("LLM experience is core to the JD", "absolutely need"),
+    "fine-tuning": ("Fine-tuning experience aligns with the JD's LLM needs", "absolutely need"),
+    "lora": ("LoRA experience aligns with the JD's fine-tuning expectations", "absolutely need"),
+    "qlora": ("QLoRA experience aligns with the JD's fine-tuning expectations", "absolutely need"),
+    "peft": ("PEFT experience aligns with the JD's fine-tuning expectations", "absolutely need"),
+    "prompt engineering": ("Prompt engineering is core to the JD's LLM work", "absolutely need"),
+    "kubernetes": ("Kubernetes experience aligns with the JD's production deployment", "nice to have"),
+    "docker": ("Docker experience aligns with the JD's production deployment", "nice to have"),
+    "aws": ("AWS experience is bonus for the JD", "nice to have"),
+    "gcp": ("GCP experience is bonus for the JD", "nice to have"),
+    "azure": ("Azure experience is bonus for the JD", "nice to have"),
+    "mlops": ("MLOps experience aligns with the JD's production needs", "nice to have"),
+    "triton": ("Triton experience aligns with the JD's inference infrastructure", "nice to have"),
+    "vector database": ("Vector database experience is core to the JD", "absolutely need"),
+    "nlp": ("NLP experience is core to the JD", "absolutely need"),
+}
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+PRODUCT_COMPANIES = set(_COMPANY_CONTEXT.keys())  # aliases for tier detection
+
+IT_SERVICES = {
+    "TCS", "Infosys", "Wipro", "HCL", "Tech Mahindra", "Cognizant",
+    "Capgemini", "Accenture", "Mindtree", "L&T Infotech", "Mphasis",
+    "Persistent", "Zensar", "Hexaware", "Birlasoft", "Cyient",
+}
+
+
+def _fmt_skill(s: str) -> str:
+    s = (s or "").strip()
+    if not s:
+        return ""
+    return s
+
+
+def _canon_skill_local(name: str) -> str:
+    """Local alias canonicalization for the reasoning template (kept in sync
+    with redrob.features.skill_features._SKILL_ALIAS)."""
+    n = (name or "").strip().lower()
+    if not n:
+        return ""
+    # Map common variants to their canonical form
+    aliases = {
+        "pgvector": "vector database",
+        "vector search": "embeddings",
+        "semantic search": "embeddings",
+        "fine-tuning llms": "fine-tuning",
+        "llm fine-tuning": "fine-tuning",
+        "instruction tuning": "fine-tuning",
+        "search relevance": "bm25",
+        "relevance ranking": "learning to rank",
+        "sentence-transformer": "sentence transformers",
+        "sbert": "sentence transformers",
+        "dense passage retrieval": "dense retrieval",
+        "natural language processing": "nlp",
+    }
+    return aliases.get(n, n)
+
+
+def _top_skills(df_row: pd.Series, k: int = 4) -> List[str]:
+    """Top JD-aligned skills from the candidate's profile.
+
+    Uses skill-alias canonicalization so that e.g. "pgvector" matches
+    the JD's "vector database" requirement.
+    """
+    skills_full = df_row.get("skills")
+    if skills_full is None:
+        skills_full = []
+    elif hasattr(skills_full, "tolist"):
+        skills_full = skills_full.tolist()
+    core_lc = {_canon_skill_local(s) for s in config.CORE_COMPETENCIES}
+    adj_lc = {_canon_skill_local(s) for s in config.ADJACENT_COMPETENCIES}
+    ranked: List[tuple] = []
+    for s in skills_full:
+        if not isinstance(s, dict):
+            continue
+        n = (s.get("name") or "").strip()
+        if not n:
+            continue
+        nl_canon = _canon_skill_local(n)
+        if nl_canon in core_lc:
+            ranked.append((0, n))
+        elif nl_canon in adj_lc:
+            ranked.append((1, n))
+        else:
+            prof = s.get("proficiency", "")
+            if prof in ("expert", "advanced"):
+                ranked.append((2, n))
+    ranked.sort()
+    return [n for _, n in ranked[:k]]
+
+
+def _skill_jd_meaning(skill: str) -> Tuple[str, str]:
+    """Why this skill matters for the JD's 'absolutely need' list."""
+    return _JD_SKILL_MEANING.get(skill.lower(), ("", ""))
+
+
+def _top_skills_with_meaning(df_row: pd.Series, k: int = 4) -> List[Tuple[str, Tuple[str, str]]]:
+    """Top JD-aligned skills paired with their JD-meaning tuple."""
+    out: List[Tuple[str, Tuple[str, str]]] = []
+    for s in _top_skills(df_row, k=k):
+        meaning = _skill_jd_meaning(s)
+        if meaning and meaning[0]:
+            out.append((s, meaning))
+    return out[:k]
+
+
+def _assessment_str(df_row: pd.Series) -> str:
+    sig = df_row.get("signals")
+    if sig is None:
+        sig = {}
+    elif hasattr(sig, "tolist"):
+        sig = sig.tolist() if sig else {}
+    if not isinstance(sig, dict):
+        return ""
+    sas = sig.get("skill_assessment_scores") or {}
+    vals = [(k, v) for k, v in sas.items()
+            if v is not None and isinstance(v, (int, float))]
+    if not vals:
+        return ""
+    top = sorted(vals, key=lambda x: -x[1])[:3]
+    return ", ".join(f"{k}={int(v)}" for k, v in top)
+
+
+def _career_evidence_snippet(df_row: pd.Series) -> str:
+    """Extract one shipped/built/launched verb from career descriptions."""
+    career = df_row.get("career")
+    if career is None:
+        career = []
+    elif hasattr(career, "tolist"):
+        career = career.tolist()
+    if not career:
+        return ""
+    verb_re = re.compile(
+        r"\b(built|shipped|launched|migrated|scaled|deployed|"
+        r"productionized|productionised|architected|led|owned|drove|"
+        r"designed|implemented|integrated|rolled out)\b",
+        re.IGNORECASE,
+    )
+    target_re = re.compile(
+        r"\b(search|ranking|retrieval|recommender|recommendation|ranker|"
+        r"vector|embedding|llm|fine.tun|rag|model|system|pipeline|"
+        r"platform|infrastructure)\w*\b",
+        re.IGNORECASE,
+    )
+    for ch in career[-3:]:
+        desc = (ch.get("description") or "")
+        if not desc:
+            continue
+        # Find first verb-noun pair
+        for sent in re.split(r"[.;]\s*", desc):
+            v = verb_re.search(sent)
+            t = target_re.search(sent)
+            if v and t:
+                return f"{v.group(0).lower()} {t.group(0).lower()}"
+        # Fallback: any verb
+        v = verb_re.search(desc)
+        if v:
+            return f"{v.group(0).lower()} applied-ML systems"
+    return ""
+
+
+def _yoe_band(yoe: float) -> str:
+    if yoe < config.YOE_IDEAL_LOW:
+        return (f"{yoe:.1f} yrs "
+                f"(slightly under JD's {config.YOE_IDEAL_LOW}-{config.YOE_IDEAL_HIGH} sweet spot)")
+    if yoe > config.YOE_IDEAL_HIGH:
+        return (f"{yoe:.1f} yrs "
+                f"(slightly over JD's {config.YOE_IDEAL_LOW}-{config.YOE_IDEAL_HIGH} sweet spot)")
+    return (f"{yoe:.1f} yrs "
+            f"(in JD's {config.YOE_IDEAL_LOW}-{config.YOE_IDEAL_HIGH} sweet spot)")
+
+
+def _location_str(df_row: pd.Series) -> str:
+    loc = (df_row.get("location") or "").strip()
+    country = (df_row.get("country") or "").strip()
+    s = f"{loc}, {country}" if loc and country else (loc or country or "")
+    return s
+
+
+def _location_fit(df_row: pd.Series) -> str:
+    """Does the candidate's location match the JD's Pune/Noida preference?"""
+    loc = (df_row.get("location") or "").lower()
+    if not loc:
+        return ""
+    preferred = [c.lower() for c in config.PREFERRED_LOCATIONS]
+    for p in preferred:
+        if p.lower() in loc:
+            return "preferred"
+    return ""
+
+
+def _is_services_company(company: str) -> bool:
+    cl = (company or "").lower()
+    for s in IT_SERVICES:
+        if s.lower() in cl:
+            return True
+    return False
+
+
+def _company_tier(df_row: pd.Series) -> str:
+    company = (df_row.get("current_company") or "").strip()
+    if not company:
+        return "unknown"
+    # Tier-3
+    for c in PRODUCT_COMPANIES:
+        if c.lower() in company.lower():
+            return "tier3"
+    # Services
+    if _is_services_company(company):
+        return "services"
+    return "tier1"
+
+
+def _rank_band(rank: int) -> str:
+    if rank <= 3:
+        return "elite"
+    if rank <= 10:
+        return "top"
+    if rank <= 50:
+        return "mid"
+    return "tail"
+
+
+# ---------------------------------------------------------------------------
+# Clause builders
+# ---------------------------------------------------------------------------
+
+def _clause_identity(df_row: pd.Series, rank: int) -> str:
+    """Clause 1: who they are + career evidence (replaces generic praise)."""
+    title = (df_row.get("current_title") or "").strip() or "this candidate"
+    company = (df_row.get("current_company") or "").strip()
+    company_ctx = _company_context(company)
+    yoe = float(df_row.get("yoe") or 0.0)
+    band = _rank_band(rank)
+    seniority = _detect_seniority(title)
+    evidence = _career_evidence_snippet(df_row)
+
+    # Drop the seniority prefix if the title itself already includes the word
+    title_lower = title.lower()
+    if seniority and seniority in title_lower:
+        prefix = ""
+    elif seniority == "principal":
+        prefix = "Principal-level "
+    elif seniority == "staff":
+        prefix = "Staff-level "
+    elif seniority == "lead":
+        prefix = "Lead-level "
+    elif seniority == "senior":
+        prefix = "Senior-level "
+    else:
+        prefix = ""
+
+    title_phrase = f"{prefix}'{title}'" if prefix else f"'{title}'"
+    ctx_phrase = f" ({company_ctx})" if company_ctx else ""
+    head = f"{title_phrase} at {company}{ctx_phrase} with {_yoe_band(yoe)}"
+
+    # Append a career-evidence fragment when available
+    if evidence:
+        return head + f" — career history shows they have {evidence}."
+
+    return head + "."
+
+
+def _strip_dup_prefix(title: str, seniority: str) -> str:
+    """If the title already contains the seniority word, don't prefix it."""
+    if not seniority:
+        return title
+    if re.search(rf"\b{seniority}\b", title, re.IGNORECASE):
+        return title
+    return title
+
+
+def _detect_seniority(title: str) -> str:
+    t = title.lower()
+    if re.search(r"\bprincipal\b", t): return "principal"
+    if re.search(r"\bstaff\b", t): return "staff"
+    if re.search(r"\b(head|chief|director|architect)\b", t): return "staff"
+    if re.search(r"\b(lead|principal)\b", t): return "lead"
+    if re.search(r"\bsenior\b", t): return "senior"
+    if re.search(r"\b(junior|jr|intern|trainee)\b", t): return "junior"
+    return ""
+
+
+def _clause_skill_jd_alignment(df_row: pd.Series) -> str:
+    """Clause 2: connect top skills to specific JD requirements."""
+    pairs = _top_skills_with_meaning(df_row, k=3)
+    if not pairs:
+        # Fallback: just name top skills without meaning
+        skills = _top_skills(df_row, k=3)
+        if skills:
+            return f"Skills intersecting the JD's stack: {', '.join(skills)}."
+        return "Limited JD-skill overlap; included for retrieval breadth."
+
+    parts = []
+    for skill, meaning in pairs:
+        why = meaning[0] if meaning and meaning[0] else ""
+        if why:
+            # Use only the first short fragment (before first comma)
+            short = why.split(",")[0].strip()
+            parts.append(f"{skill} ({short})")
+        else:
+            parts.append(skill)
+    return "Matches JD 'absolutely need' on: " + ", ".join(parts) + "."
+
+
+def _clause_assessment(df_row: pd.Series) -> str:
+    """Clause 3: Redrob skill assessment scores (high-value when present)."""
+    assess = _assessment_str(df_row)
+    if not assess:
+        return "No Redrob skill-assessment scores on file (verified skills via profile only)."
+    return f"Redrob skill assessments: {assess}."
+
+
+def _clause_career_coherence(df_row: pd.Series, rank: int) -> str:
+    """Clause 4 (mid/tail): career-graph / community coherence."""
+    car_ev = float(df_row.get("career_evidence") or 0.0)
+    car_coh = float(df_row.get("career_coherence") or 0.0)
+    if car_ev >= 0.5:
+        return "Career-graph evidence: descriptions reference shipped production ranking/retrieval systems."
+    if car_coh >= 0.7:
+        return "Career-coherence is high: titles form a consistent applied-ML progression."
+    return "Included on retrieval + skill signals (career descriptions lack shipped-system keywords)."
+
+
+def _clause_geo_notice(df_row: pd.Series) -> str:
+    """Clause 5: location fit + notice period context."""
+    loc = _location_fit(df_row)
+    notice = int(df_row.get("notice_period_days") or 0)
+    reloc = bool(int(df_row.get("willing_to_relocate") or 0))
+
+    geo_part = ""
+    if loc == "preferred":
+        loc_str = _location_str(df_row)
+        geo_part = f"Located in JD-preferred city ({loc_str}); relocation not required."
+    elif reloc:
+        geo_part = f"Willing to relocate to a JD-preferred city (current: {_location_str(df_row)})."
+
+    notice_part = ""
+    if notice and notice <= 30:
+        notice_part = f"Notice period {notice} days — within the JD's sub-30-day preference."
+    elif notice and notice <= 60:
+        notice_part = f"Notice period {notice} days — within tolerance; the JD will buy out up to 30 days."
+    elif notice:
+        notice_part = f"Notice period {notice} days — above the JD's 30-day preference; bar is higher."
+
+    if geo_part and notice_part:
+        return geo_part + " " + notice_part
+    return geo_part or notice_part or ""
+
+
+def _clause_concerns(df_row: pd.Series, rank: int) -> str:
+    """Honest concerns clause."""
+    concerns: List[str] = []
+
+    notice = int(df_row.get("notice_period_days") or 0)
+    if notice >= 90:
+        concerns.append(f"{notice}-day notice period is above the JD's 30-day preference")
+
+    honeypot = float(df_row.get("honeypot_penalty") or 0.0)
+    if honeypot >= 0.25:
+        concerns.append(f"profile shows some inconsistency signals (honeypot penalty {honeypot:.2f})")
+
+    rec = float(df_row.get("recruitability") or 0.0)
+    if rec < 0.3:
+        concerns.append("recruiter-engagement signals are muted (verify interest before outreach)")
+
+    is_junior = bool(int(df_row.get("is_junior_title") or 0))
+    yoe = float(df_row.get("yoe") or 0.0)
+    in_band = config.YOE_IDEAL_LOW <= yoe <= config.YOE_IDEAL_HIGH
+    if is_junior and in_band and rank <= 50:
+        concerns.append("junior title despite senior-YOE band — verify level")
+
+    # Negative-spec flags
+    if bool(int(df_row.get("is_consulting_only") or 0)):
+        concerns.append("career is consulting-only; JD strongly prefers product-company experience")
+    if bool(int(df_row.get("is_framework_enthusiast") or 0)):
+        concerns.append("skill mix is framework-heavy (LangChain/LlamaIndex) without deep retrieval fundamentals")
+
+    if not concerns:
+        return "No major concerns."
+    if len(concerns) == 1:
+        return f"Honest note: {concerns[0]}."
+    if len(concerns) == 2:
+        return f"Honest notes: {concerns[0]}; {concerns[1]}."
+    return "Honest notes: " + "; ".join(concerns[:2]) + f"; and {len(concerns)-2} more."
+
+
+# ---------------------------------------------------------------------------
+# Template assembly per rank-band × company-tier
+# ---------------------------------------------------------------------------
+
+def _assemble(df_row: pd.Series, rank: int, band: str, tier: str) -> List[str]:
+    """Return 3-4 clauses appropriate for this rank band and company tier."""
+    clauses: List[str] = []
+    band_tier = f"{band}_{tier}"
+    company = (df_row.get("current_company") or "").strip()
+
+    # ---- Clause 1: identity + career evidence ----
+    c1 = _clause_identity(df_row, rank)
+    clauses.append(c1)
+
+    # ---- Clause 2: skill → JD alignment ----
+    clauses.append(_clause_skill_jd_alignment(df_row))
+
+    # ---- Clause 3: assessment scores (top half only) ----
+    if rank <= 50:
+        clauses.append(_clause_assessment(df_row))
+
+    # ---- Clause 4: geo + notice (top half only) ----
+    geo_notice = _clause_geo_notice(df_row)
+    if geo_notice and rank <= 50:
+        clauses.append(geo_notice)
+
+    # ---- Clause 5: career-graph / coherence (mid/tail) ----
+    if band in ("mid", "tail"):
+        clauses.append(_clause_career_coherence(df_row, rank))
+
+    # ---- Clause 6: concerns ----
+    if rank <= 100:
+        # Always include a concerns clause
+        clauses.append(_clause_concerns(df_row, rank))
+
+    return clauses
+
+
+def _choose_clauses(rank: int, df_row: pd.Series) -> List[str]:
+    """Choose clauses appropriate for this rank and candidate."""
+    band = _rank_band(rank)
+    tier = _company_tier(df_row)
+    return _assemble(df_row, rank, band, tier)
+
+
+# ---------------------------------------------------------------------------
+# Public entry points
+# ---------------------------------------------------------------------------
+
+def generate_reasoning(df_row: pd.Series, rank: int, n_total: int = 100) -> str:
+    """Compose a 3-5 clause reasoning string for one candidate."""
+    clauses = _choose_clauses(rank, df_row)
+    # Ensure 3-5 clauses
+    while len(clauses) < 3:
+        clauses.append("No major concerns.")
+    clauses = clauses[:5]
+    text = " ".join(clauses)
+    text = text.replace("\n", " ").replace("\r", " ")
+    text = text.replace('"', "'")
+    # Trim if too long (validator only requires correct columns, but very long
+    # reasoning hurts Stage 4 readability)
+    if len(text) > 800:
+        text = text[:797] + "..."
+    return text
+
+
+def generate_reasoning_dataframe(df: pd.DataFrame) -> List[str]:
+    """Return a list of reasoning strings, one per row, in input order."""
+    out: List[str] = []
+    n = len(df)
+    for i in range(n):
+        out.append(generate_reasoning(df.iloc[i], i + 1, n))
+    return out

redrob/retrieval/__init__.py

@@ -0,0 +1,10 @@
+"""Hybrid retrieval layer: BM25 + sentence-transformers + structured gate + RRF."""
+from .bm25 import build_bm25, bm25_search
+from .dense import build_dense_index, dense_search, dense_model_name
+from .rrf import rrf_fuse
+
+__all__ = [
+    "build_bm25", "bm25_search",
+    "build_dense_index", "dense_search", "dense_model_name",
+    "rrf_fuse",
+]

redrob/retrieval/bm25.py

@@ -0,0 +1,83 @@
+"""BM25 sparse retrieval over candidate text corpora."""
+from __future__ import annotations
+
+import pickle
+import re
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+
+from .. import config
+from ..data import _tokenize, load_or_build_parquet
+
+
+def _query_tokens(query: str) -> List[str]:
+    return _tokenize(query)
+
+
+def build_bm25(force: bool = False) -> Tuple[object, List[str]]:
+    """Build (or load) the BM25 index over the parquet corpus."""
+    if not force and config.BM25_PICKLE.exists() and config.BM25_TOKENS_PICKLE.exists():
+        with open(config.BM25_PICKLE, "rb") as f:
+            bm25 = pickle.load(f)
+        with open(config.BM25_TOKENS_PICKLE, "rb") as f:
+            tokens_list = pickle.load(f)
+        return bm25, tokens_list
+
+    from rank_bm25 import BM25Okapi  # type: ignore
+
+    df = load_or_build_parquet()
+    tokens_list: List[List[str]] = []
+    for txt in df["text_corpus"].tolist():
+        tokens_list.append(_tokenize(txt or ""))
+
+    bm25 = BM25Okapi(tokens_list)
+
+    with open(config.BM25_PICKLE, "wb") as f:
+        pickle.dump(bm25, f, protocol=pickle.HIGHEST_PROTOCOL)
+    with open(config.BM25_TOKENS_PICKLE, "wb") as f:
+        pickle.dump(tokens_list, f, protocol=pickle.HIGHEST_PROTOCOL)
+    return bm25, tokens_list
+
+
+def bm25_search(
+    queries: List[str],
+    top_k: int = config.BM25_TOP_K,
+    bm25_obj: Optional[Tuple[object, List[str]]] = None,
+    candidate_ids: Optional[List[str]] = None,
+) -> Dict[str, List[Tuple[str, float]]]:
+    """Run BM25 for each query and return top-k per query.
+
+    Returns a dict mapping query -> list of (candidate_id, score).
+    """
+    bm25, _ = bm25_obj or build_bm25()
+    df = load_or_build_parquet()
+    if candidate_ids is None:
+        candidate_ids = df["candidate_id"].tolist()
+    n_docs = len(candidate_ids)
+    out: Dict[str, List[Tuple[str, float]]] = {}
+    for q in queries:
+        qtoks = _query_tokens(q)
+        if not qtoks:
+            out[q] = []
+            continue
+        scores = bm25.get_scores(qtoks)
+        # Top-k by argpartition (O(n)) + small sort
+        if top_k >= n_docs:
+            order = np.argsort(-scores)
+        else:
+            idx = np.argpartition(-scores, top_k)[:top_k]
+            order = idx[np.argsort(-scores[idx])]
+        out[q] = [(candidate_ids[int(i)], float(scores[int(i)])) for i in order]
+    return out
+
+
+if __name__ == "__main__":
+    bm25, toks = build_bm25(force=False)
+    print(f"BM25 docs={len(toks):,}")
+    res = bm25_search(["PyTorch transformers LoRA RAG vector search"], top_k=10)
+    for q, hits in res.items():
+        print(q)
+        for cid, sc in hits[:10]:
+            print(f"  {cid}\t{sc:.3f}")

redrob/retrieval/dense.py

@@ -0,0 +1,139 @@
+"""Dense retrieval using sentence-transformers.
+
+The model is configurable via env var REDROB_DENSE_MODEL.
+Stronger BGE variants are supported when they are available locally.
+"""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+
+from .. import config
+from ..data import load_or_build_parquet
+
+
+_DENSE_MODEL_CACHE: Dict[str, object] = {}
+
+
+def dense_model_name() -> str:
+    """Resolve the dense model name from env, with validation."""
+    name = os.environ.get("REDROB_DENSE_MODEL", config.DEFAULT_DENSE_MODEL).strip()
+    if name and name not in config.DENSE_MODEL_OPTIONS:
+        # Allow any model name the user has cached; warn in logs.
+        pass
+    return name or config.DEFAULT_DENSE_MODEL
+
+
+def _load_model():
+    name = dense_model_name()
+    if name in _DENSE_MODEL_CACHE:
+        return _DENSE_MODEL_CACHE[name]
+    from sentence_transformers import SentenceTransformer  # type: ignore
+    # Try the requested model. If it isn't cached and we're offline,
+    # fall back to MiniLM (always cached at construction time below).
+    candidates = [name] + [m for m in config.DENSE_MODEL_OPTIONS if m != name]
+    last_err: Optional[Exception] = None
+    for cand in candidates:
+        try:
+            model = SentenceTransformer(cand, cache_folder=str(config.MODELS_DIR))
+            _DENSE_MODEL_CACHE[name] = model
+            return model
+        except Exception as e:  # noqa: BLE001
+            last_err = e
+            continue
+    raise RuntimeError(f"Could not load any sentence-transformers model: {last_err}")
+
+
+def _model_dim() -> int:
+    m = _load_model()
+    return int(m.get_sentence_embedding_dimension())
+
+
+def build_dense_index(force: bool = False, candidate_ids: Optional[List[str]] = None) -> Tuple[np.ndarray, List[str]]:
+    """Encode every candidate's text corpus and return (matrix, ids).
+
+    If `candidate_ids` is provided, only those candidates are encoded.
+    The full 100k corpus is encoded on first call; subsequent runs use
+    the cached matrix and align to the candidate_ids subset.
+    """
+    if not force and config.DENSE_NPY.exists() and config.DENSE_IDS_JSON.exists():
+        emb_all = np.load(config.DENSE_NPY)
+        ids_all = json.loads(config.DENSE_IDS_JSON.read_text(encoding="utf-8"))
+        if candidate_ids is None:
+            return emb_all, ids_all
+        # subset to requested ids
+        idx_map = {c: i for i, c in enumerate(ids_all)}
+        idxs = [idx_map[c] for c in candidate_ids if c in idx_map]
+        return emb_all[idxs], [ids_all[i] for i in idxs]
+
+    model = _load_model()
+    df = load_or_build_parquet()
+    if candidate_ids is not None:
+        df = df[df["candidate_id"].isin(set(candidate_ids))].reset_index(drop=True)
+    texts = df["text_corpus"].fillna("").tolist()
+    # truncate to model max seq length * 6 chars
+    texts = [(t or "")[:2000] for t in texts]
+    embeddings = model.encode(
+        texts,
+        batch_size=256,
+        show_progress_bar=False,
+        convert_to_numpy=True,
+        normalize_embeddings=True,
+    ).astype(np.float32)
+
+    ids = df["candidate_id"].tolist()
+    np.save(config.DENSE_NPY, embeddings)
+    config.DENSE_IDS_JSON.write_text(json.dumps(ids), encoding="utf-8")
+    return embeddings, ids
+
+
+def dense_search(
+    queries: List[str],
+    top_k: int = config.DENSE_TOP_K,
+    matrix: Optional[Tuple[np.ndarray, List[str]]] = None,
+    candidate_ids: Optional[List[str]] = None,
+) -> Dict[str, List[Tuple[str, float]]]:
+    """Run dense retrieval.
+
+    If a precomputed matrix is passed it is used. Otherwise the
+    candidate_ids subset (or full corpus) is encoded and the matrix
+    returned for reuse.
+    """
+    if matrix is None:
+        matrix = build_dense_index(candidate_ids=candidate_ids)
+    emb, ids = matrix
+
+    model = _load_model()
+    qvecs = model.encode(
+        queries,
+        batch_size=64,
+        show_progress_bar=False,
+        convert_to_numpy=True,
+        normalize_embeddings=True,
+    ).astype(np.float32)
+
+    out: Dict[str, List[Tuple[str, float]]] = {}
+    for q, qv in zip(queries, qvecs):
+        scores = emb @ qv  # cosine because vectors are normalised
+        if top_k >= len(scores):
+            order = np.argsort(-scores)
+        else:
+            idx = np.argpartition(-scores, top_k)[:top_k]
+            order = idx[np.argsort(-scores[idx])]
+        out[q] = [(ids[int(i)], float(scores[int(i)])) for i in order]
+    return out
+
+
+if __name__ == "__main__":
+    print("model:", dense_model_name())
+    m, ids = build_dense_index(force=False)
+    print("emb shape:", m.shape, "ids:", len(ids))
+    res = dense_search(["Senior AI Engineer with hybrid search and RAG experience"], top_k=5)
+    for q, hits in res.items():
+        print(q)
+        for cid, sc in hits:
+            print(f"  {cid}\t{sc:.4f}")

redrob/retrieval/rrf.py

@@ -0,0 +1,45 @@
+"""Reciprocal Rank Fusion of multiple ranked lists."""
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Dict, Iterable, List, Tuple
+
+
+def rrf_fuse(
+    rankings: List[List[Tuple[str, float]]],
+    k: int = 60,
+    top_n: int | None = None,
+    channel_weights: List[float] | None = None,
+) -> List[Tuple[str, float, Dict[str, int]]]:
+    """Combine multiple ranked lists using weighted RRF.
+
+    Each ranking is a list of (id, score). Higher score is better.
+    channel_weights lets you weight some channels more (e.g. BM25 1.2,
+    dense 0.8). Returns a single fused list of (id, rrf_score, per_channel_rank).
+    """
+    fused: Dict[str, float] = defaultdict(float)
+    ranks: Dict[str, Dict[str, int]] = defaultdict(dict)
+    n_channels = len(rankings)
+    if channel_weights is None:
+        channel_weights = [1.0] * n_channels
+    assert len(channel_weights) == n_channels, "channel_weights must match rankings"
+    for ch_idx, (ranking, w) in enumerate(zip(rankings, channel_weights)):
+        ch_name = f"ch{ch_idx}"
+        for r, (cid, _score) in enumerate(ranking, start=1):
+            fused[cid] += w / (k + r)
+            ranks[cid][ch_name] = r
+    items = sorted(fused.items(), key=lambda x: -x[1])
+    out: List[Tuple[str, float, Dict[str, int]]] = []
+    for cid, s in items:
+        out.append((cid, float(s), ranks[cid]))
+        if top_n is not None and len(out) >= top_n:
+            break
+    return out
+
+
+if __name__ == "__main__":
+    a = [("A", 1.0), ("B", 0.9), ("C", 0.8)]
+    b = [("B", 0.95), ("A", 0.7), ("D", 0.5)]
+    fused = rrf_fuse([a, b], k=60)
+    for cid, s, r in fused:
+        print(cid, s, r)

redrob/submit/__init__.py

@@ -0,0 +1,4 @@
+"""Submission writer with validation hook."""
+from .write import write_submission
+
+__all__ = ["write_submission"]

redrob/submit/write.py

@@ -0,0 +1,91 @@
+"""Write the final submission CSV and (optionally) run the validator."""
+from __future__ import annotations
+
+import csv
+import subprocess
+from pathlib import Path
+from typing import List
+
+import pandas as pd
+
+from .. import config
+
+
+def _normalise_score(s: float) -> float:
+    """Squeeze score to [0, 1] in a monotonic way that preserves relative order."""
+    if s <= 0:
+        return 0.0
+    return float(1 / (1 + 2.71828 ** (-0.6 * s)))
+
+
+def write_submission(
+    df_ranked: pd.DataFrame,
+    out_path: Path | str = config.DEFAULT_OUTPUT_CSV,
+    validate: bool = True,
+    validator: Path | str | None = config.DEFAULT_VALIDATOR,
+) -> Path:
+    """Write the final CSV and (optionally) invoke the validator.
+
+    Expects `df_ranked` to be sorted best-first, with columns
+    `candidate_id` and `final_score`. Reasoning is generated from the
+    candidate row.
+
+    The validator requires that scores be *strictly* non-increasing
+    (since the tie-breaker is candidate_id ascending on equal scores).
+    We use the raw LightGBM score and apply a 1e-4 epsilon (well below
+    the model's scoring resolution but strictly decreasing in float
+    comparison). The displayed 6-decimal value preserves the model's
+    ranking resolution.
+    """
+    from ..reasoning.template import generate_reasoning_dataframe
+
+    out_path = Path(out_path)
+    df_ranked = df_ranked.reset_index(drop=True).copy()
+    if len(df_ranked) > 100:
+        df_ranked = df_ranked.head(100)
+
+    # Build a strictly decreasing score for each row.
+    # Use the raw final_score (LightGBM LambdaRank produces values in
+    # roughly [-15, +15]). Apply a per-index epsilon of 1e-4 which
+    # guarantees strict float-decrease while being invisible at 6 decimals.
+    norm_scores = df_ranked["final_score"].astype(float).tolist()
+    eps = 1e-4
+    for i in range(1, len(norm_scores)):
+        if norm_scores[i] >= norm_scores[i - 1] - eps * (i - 1):
+            norm_scores[i] = norm_scores[i - 1] - eps * i
+
+    # Reasoning
+    reasonings = generate_reasoning_dataframe(df_ranked)
+
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w", encoding="utf-8", newline="") as f:
+        w = csv.writer(f, quoting=csv.QUOTE_MINIMAL)
+        w.writerow(["candidate_id", "rank", "score", "reasoning"])
+        for i, row in df_ranked.iterrows():
+            cid = str(row["candidate_id"])
+            score = f"{norm_scores[i]:.6f}"
+            w.writerow([cid, str(i + 1), score, reasonings[i]])
+
+    if validate and validator and Path(validator).exists():
+        try:
+            r = subprocess.run(
+                ["python", str(validator), str(out_path)],
+                capture_output=True, text=True, check=False,
+            )
+            if r.returncode == 0:
+                print(f"[validator] OK: {out_path}")
+            else:
+                print(f"[validator] FAILED:\n{r.stdout}\n{r.stderr}")
+        except Exception as e:
+            print(f"[validator] error: {e}")
+
+    return out_path
+
+
+if __name__ == "__main__":
+    from ..data import load_or_build_parquet
+    from ..rank.predict import rank_shortlist
+    df = load_or_build_parquet().sample(120, random_state=0)
+    ranked = rank_shortlist(df, use_lgbm=False)
+    p = write_submission(ranked, validate=True)
+    print("wrote:", p)

requirements.txt

@@ -1,3 +1,11 @@
-altair
-pandas
-streamlit
+streamlit>=1.30
+pandas>=2.0
+numpy>=1.24
+scikit-learn>=1.3
+lightgbm>=4.0
+networkx>=3.1
+ijson>=3.2
+rank_bm25>=0.2.2
+pyyaml>=6.0
+tqdm>=4.65
+sentence-transformers>=2.6

scripts/run_ranking.py

@@ -0,0 +1,317 @@
+"""End-to-end ranking pipeline.
+
+This is THE reproduce command listed in submission_metadata.yaml.
+
+Usage:
+    python scripts/run_ranking.py \\
+        --candidates "<path-to-candidates.jsonl>" \\
+        --out        "<path-to-submission.csv>"
+
+It streams the candidate pool, builds BM25 + (optional) dense indices,
+fuses them with RRF, computes the full feature set on the shortlist, and
+applies the trained ranker (or the deterministic fallback). The result
+is a 100-row CSV that the validator checks for compliance.
+
+If a precomputed artifact is missing, the pipeline rebuilds it. All
+artifacts live under `artifacts/`.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import pickle
+import re
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+# Determinism
+os.environ.setdefault("PYTHONHASHSEED", "0")
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+
+from redrob import config
+from redrob.blueprint import build_blueprint, save_blueprint
+from redrob.data import load_or_build_parquet
+from redrob.retrieval.bm25 import build_bm25, bm25_search
+from redrob.retrieval.dense import (
+    build_dense_index, dense_search, dense_model_name,
+)
+from redrob.retrieval.rrf import rrf_fuse
+from redrob.rank.predict import rank_shortlist
+from redrob.submit.write import write_submission
+
+
+def _save_id_map(path: Path, ids: list) -> None:
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(ids, f)
+
+
+def _load_id_map(path: Path) -> list:
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def run(args) -> None:
+    t_start = time.time()
+    print(f"[{time.time()-t_start:5.1f}s] starting pipeline")
+
+    # 1) Blueprint
+    save_blueprint()
+    bp = build_blueprint()
+    queries = bp["query_terms"]
+    print(f"[{time.time()-t_start:5.1f}s] blueprint saved ({len(config.CORE_COMPETENCIES)} core skills)")
+
+    # 2) Load or build the canonicalised parquet
+    df = load_or_build_parquet(args.candidates)
+    print(f"[{time.time()-t_start:5.1f}s] canonicalised: {len(df):,} candidates")
+
+    # 3) BM25 sparse retrieval
+    bm25, _ = build_bm25(force=False)
+    sparse_hits = bm25_search(queries, top_k=config.BM25_TOP_K, bm25_obj=(bm25, _))
+    sparse_rankings = list(sparse_hits.values())
+    print(f"[{time.time()-t_start:5.1f}s] BM25 ready (top-{config.BM25_TOP_K} per query)")
+
+    # 4) Dense retrieval (or fallback)
+    dense_rankings: list = []
+    if not args.no_dense:
+        try:
+            t_dense = time.time()
+            print(f"[{time.time()-t_start:5.1f}s] building dense index (model={dense_model_name()}) ...")
+            # If the cache exists, use it (fast). Otherwise encode and cache.
+            emb, ids = build_dense_index(force=False)
+            dense_hits = dense_search(queries, top_k=config.DENSE_TOP_K, matrix=(emb, ids))
+            dense_rankings = list(dense_hits.values())
+            print(f"[{time.time()-t_start:5.1f}s] dense ready in {time.time()-t_dense:.1f}s")
+        except Exception as e:  # noqa: BLE001
+            print(f"[{time.time()-t_start:5.1f}s] dense unavailable ({e}); using BM25-only")
+            dense_rankings = []
+    else:
+        print(f"[{time.time()-t_start:5.1f}s] dense path disabled by --no_dense")
+
+    # 5) Structured high-recall gate as SOFT score (saturating)
+    gate_terms_lc = {t.lower() for t in config.STRUCTURED_GATE_TERMS}
+    df_reset = df.reset_index(drop=True)
+
+    def _gate_text_score(text):
+        if not text:
+            return 0
+        tl = text.lower()
+        n_hits = sum(1 for g in gate_terms_lc if g in tl)
+        return float(min(1.0, n_hits / 5.0))
+
+    def _gate_skill_score(skills):
+        if skills is None:
+            return 0.0
+        if hasattr(skills, "tolist"):
+            skills = skills.tolist()
+        n = sum(1 for s in (skills or []) if (s or "").lower() in gate_terms_lc)
+        return float(min(1.0, n / 3.0))
+
+    text_scores = df_reset["text_corpus"].fillna("").apply(_gate_text_score).to_numpy()
+    skill_scores = df_reset["skill_names"].apply(_gate_skill_score).to_numpy()
+    gate_score = np.maximum(text_scores, skill_scores)
+    structured_hit = (gate_score > 0.0).astype(np.float32)
+    print(f"[{time.time()-t_start:5.1f}s] structured gate: median score {np.median(gate_score):.2f}, "
+          f"hits {int(structured_hit.sum()):,}/{len(df):,}")
+
+    # 6) RRF over all channels with per-channel weights
+    # BM25 (reliable) > structured-gate (binary OK) > dense (noisy in this domain)
+    rankings = list(sparse_rankings) + list(dense_rankings)
+    if dense_rankings:
+        channel_weights = [1.2] * len(sparse_rankings) + [0.8] * len(dense_rankings)
+    else:
+        channel_weights = [1.0] * len(sparse_rankings)
+    fused = rrf_fuse(rankings, k=config.RRF_K, top_n=config.SHORTLIST_N,
+                     channel_weights=channel_weights)
+    shortlist_ids = [cid for cid, _score, _r in fused]
+    shortlist_set = set(shortlist_ids)
+    if args.verbose:
+        print(f"[{time.time()-t_start:5.1f}s] RRF shortlist: {len(shortlist_ids):,}")
+
+    # Add top structured-gate candidates not already in shortlist (sorted by gate_score)
+    struct_candidates = df_reset.loc[gate_score > 0.0].copy()
+    struct_candidates = struct_candidates.assign(_g=gate_score[gate_score > 0.0])
+    struct_candidates = struct_candidates.sort_values("_g", ascending=False)
+    added = 0
+    for cid in struct_candidates["candidate_id"].tolist():
+        if cid not in shortlist_set:
+            shortlist_ids.append(cid)
+            shortlist_set.add(cid)
+            added += 1
+            if added >= args.structured_top_up:
+                break
+    if args.verbose:
+        print(f"[{time.time()-t_start:5.1f}s] +{added:,} from structured gate (total shortlist: {len(shortlist_ids):,})")
+    struct_ids = df_reset.loc[gate_score > 0.0, "candidate_id"].tolist()
+    gate_score_map = {cid: float(s) for cid, s in zip(df_reset["candidate_id"].tolist(), gate_score)}
+
+    # ----------------------------------------------------------------------
+    # Plain-language Tier-5 retrieval top-up (added by super-smart-agent review).
+    # Some excellent candidates don't say RAG/Pinecone in skills, but their
+    # career DESCRIPTIONS mention shipping ranking/search/recsys systems.
+    # Optimised: pre-compile regexes, use only the first N rows for speed,
+    # and score on text_corpus (already-built column) instead of building
+    # new strings per candidate.
+    # ----------------------------------------------------------------------
+    _SHIP_KW = re.compile(
+        r"\b(built|shipped|launched|scaled|deployed|owned|led|designed|"
+        r"implemented|migrated|productionized)\b",
+        re.IGNORECASE,
+    )
+    _RETRIEVAL_KW = re.compile(
+        r"\b(matching|personalization|personalisation|marketplace|"
+        r"recruiter engagement|click.?through|engagement|"
+        r"index|refresh|search system|relevance|re-ranking|reranking|"
+        r"semantic|embeddings|indexing|indexed|ranker)\b",
+        re.IGNORECASE,
+    )
+    _PRODUCT_TITLE_KW = re.compile(
+        r"\b(ai engineer|machine learning|ml engineer|"
+        r"applied scientist|research engineer|search engineer|"
+        r"ranking engineer|recommendation|nlp engineer|data scientist)\b",
+        re.IGNORECASE,
+    )
+    _PLAIN_LANG_TOPUP = 200  # how many extra candidates to add
+
+    plain_lang_candidates = []
+    _current_titles = df_reset["current_title"].fillna("").str.lower().tolist()
+    _text_corpuses = df_reset["text_corpus"].fillna("").str[:1000].tolist()  # cap length
+    _cids_all = df_reset["candidate_id"].tolist()
+    n_total = len(_cids_all)
+    # Scan only candidates not already in shortlist (faster loop)
+    scan_indices = [i for i, cid in enumerate(_cids_all) if cid not in shortlist_set]
+
+    for i in scan_indices:
+        # Quick title pre-filter
+        if not _PRODUCT_TITLE_KW.search(_current_titles[i] or ""):
+            continue
+        # Use the pre-built text_corpus (head 1000 chars) — fast substring match
+        text = (_text_corpuses[i] or "").lower()
+        if not text:
+            continue
+        n_ship = len(_SHIP_KW.findall(text))
+        if n_ship < 1:
+            continue
+        n_ret = len(_RETRIEVAL_KW.findall(text))
+        if n_ret >= 1:
+            plain_lang_candidates.append((n_ship + n_ret, _cids_all[i]))
+
+    plain_lang_candidates.sort(reverse=True)
+    added = 0
+    for _, cid in plain_lang_candidates[:_PLAIN_LANG_TOPUP]:
+        if cid not in shortlist_set:
+            shortlist_ids.append(cid)
+            shortlist_set.add(cid)
+            added += 1
+    if args.verbose:
+        print(f"[{time.time()-t_start:5.1f}s] +{added:,} from plain-language top-up "
+              f"(of {len(plain_lang_candidates):,} eligible; scanned {len(scan_indices):,})")
+
+    # 7) Build the shortlist dataframe
+    shortlist_df = df_reset[df_reset["candidate_id"].isin(shortlist_set)].copy()
+
+    # 8) Per-channel scores for the shortlist (sparse/dense/rrf)
+    sparse_score_by_id: dict = {}
+    for ranking in sparse_rankings:
+        for cid, sc in ranking:
+            if cid in shortlist_set:
+                sparse_score_by_id[cid] = max(sparse_score_by_id.get(cid, 0.0), sc)
+    dense_score_by_id: dict = {}
+    for ranking in dense_rankings:
+        for cid, sc in ranking:
+            if cid in shortlist_set:
+                dense_score_by_id[cid] = max(dense_score_by_id.get(cid, 0.0), sc)
+    rrf_score_by_id: dict = {cid: s for cid, s, _r in fused}
+
+    shortlist_df["sparse_score"] = shortlist_df["candidate_id"].map(lambda c: sparse_score_by_id.get(c, 0.0))
+    shortlist_df["dense_score"] = shortlist_df["candidate_id"].map(lambda c: dense_score_by_id.get(c, 0.0))
+    shortlist_df["rrf_score"] = shortlist_df["candidate_id"].map(lambda c: rrf_score_by_id.get(c, 0.0))
+    shortlist_df["structured_hit"] = shortlist_df["candidate_id"].map(lambda c: gate_score_map.get(c, 0.0))
+
+    print(f"[{time.time()-t_start:5.1f}s] computing features for {len(shortlist_df):,} shortlist candidates")
+
+    # 9) Rank (uses LightGBM if model exists, else deterministic fallback)
+    use_lgbm = not args.no_lgbm
+    if not use_lgbm:
+        print(f"[{time.time()-t_start:5.1f}s] ranker: deterministic fallback (--no_lgbm)")
+    else:
+        print(f"[{time.time()-t_start:5.1f}s] ranker: LightGBM (auto-trains if missing)")
+    ranked = rank_shortlist(shortlist_df, use_lgbm=use_lgbm)
+
+    # ----------------------------------------------------------------------
+    # v11: Final logistics eligibility filter.
+    # JD: "Location: Pune/Noida-preferred but flexible. Outside India:
+    # case-by-case, but we don't sponsor work visas."
+    # A non-India candidate who is not willing to relocate is, for this
+    # company, functionally unreachable. Remove them from the ranked
+    # shortlist BEFORE selecting top-100, so they are replaced by the
+    # next eligible candidate from below. This does NOT change top-10
+    # or top-50 (all six removed are in ranks 51–100) and preserves any
+    # non-India candidate who IS willing to relocate.
+    # ----------------------------------------------------------------------
+    country_arr = ranked["country"].fillna("").astype(str).to_numpy()
+    reloc_arr = ranked["willing_to_relocate"].fillna(0).astype(float).to_numpy()
+    eligible_mask = ~((country_arr != "India") & (reloc_arr == 0))
+    n_removed = int((~eligible_mask).sum())
+    print(f"[{time.time()-t_start:5.1f}s] v11 eligibility filter: kept "
+          f"{int(eligible_mask.sum()):,}, removed {n_removed} non-India + not-willing-to-relocate")
+
+    # 10) Top-100 (selected from the eligible subset)
+    top = ranked[eligible_mask].head(100).copy()
+    print(f"[{time.time()-t_start:5.1f}s] top-100 selected")
+
+    # 11) Honeypot self-check
+    n_honeypot = int((top["honeypot_penalty"] >= config.HONEYPOT_HARD_EXCLUDE).sum())
+    n_low_title = int((top["title_fit_blend"] < 0.10).sum())
+    n_services_top50 = int(top.head(50)["is_services_company"].astype(int).sum()) if "is_services_company" in top.columns else 0
+    print(f"[{time.time()-t_start:5.1f}s] self-check: honeypot_hard={n_honeypot}, low_title_top100={n_low_title}, services_top50={n_services_top50}")
+
+    # 12) Write CSV
+    out = write_submission(top, out_path=args.out, validate=True)
+    print(f"[{time.time()-t_start:5.1f}s] submission written: {out}")
+    print(f"[{time.time()-t_start:5.1f}s] DONE")
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument(
+        "--candidates",
+        default=str(config.DEFAULT_CANDIDATES_JSONL),
+        help="Path to candidates.jsonl",
+    )
+    ap.add_argument(
+        "--out",
+        default=str(config.DEFAULT_OUTPUT_CSV),
+        help="Path to submission CSV",
+    )
+    ap.add_argument(
+        "--no_dense",
+        action="store_true",
+        help="Disable sentence-transformers (use BM25-only retrieval)",
+    )
+    ap.add_argument(
+        "--no_lgbm",
+        action="store_true",
+        help="Use the deterministic scorer instead of the LightGBM ranker",
+    )
+    ap.add_argument(
+        "--structured_top_up",
+        type=int,
+        default=2000,
+        help="How many structured-gate candidates to add beyond the RRF shortlist",
+    )
+    ap.add_argument("--verbose", action="store_true")
+    args = ap.parse_args()
+    run(args)
+
+
+if __name__ == "__main__":
+    main()